mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-25 23:16:26 +00:00
Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
09e6a7e6d8 | ||
|
|
5814d77b16 | ||
|
|
83773af997 | ||
|
|
bbb68f7490 | ||
|
|
b3dee13954 | ||
|
|
33ef0a6ea5 | ||
|
|
7474fd4c52 | ||
|
|
64b6f559fd | ||
|
|
0de6f6aabb |
@@ -151,6 +151,12 @@ src/
|
||||
│ ├── rate_limiter.rs # Per-tool rate limiting
|
||||
│ └── storage.rs # Linear memory persistence
|
||||
│
|
||||
├── db/ # Database abstraction layer
|
||||
│ ├── mod.rs # Database trait (~60 async methods)
|
||||
│ ├── postgres.rs # PostgreSQL backend (delegates to Store + Repository)
|
||||
│ ├── libsql_backend.rs # libSQL/Turso backend (embedded SQLite)
|
||||
│ └── libsql_migrations.rs # SQLite-dialect schema (idempotent)
|
||||
│
|
||||
├── workspace/ # Persistent memory system (OpenClaw-inspired)
|
||||
│ ├── mod.rs # Workspace struct, memory operations
|
||||
│ ├── document.rs # MemoryDocument, MemoryChunk, WorkspaceEntry
|
||||
@@ -201,6 +207,7 @@ When designing new features or systems, always prefer generic/extensible archite
|
||||
- Use `RwLock` for concurrent read/write access
|
||||
|
||||
### Traits for Extensibility
|
||||
- `Database` - Add new database backends (must implement all ~60 methods)
|
||||
- `Channel` - Add new input sources
|
||||
- `Tool` - Add new capabilities
|
||||
- `LlmProvider` - Add new LLM backends
|
||||
@@ -248,7 +255,12 @@ Pending -> InProgress -> Completed -> Submitted -> Accepted
|
||||
|
||||
Environment variables (see `.env.example`):
|
||||
```bash
|
||||
# Database backend (default: postgres)
|
||||
DATABASE_BACKEND=postgres # or "libsql" / "turso"
|
||||
DATABASE_URL=postgres://user:pass@localhost/ironclaw
|
||||
LIBSQL_PATH=~/.ironclaw/ironclaw.db # libSQL local path (default)
|
||||
# LIBSQL_URL=libsql://xxx.turso.io # Turso cloud (optional)
|
||||
# LIBSQL_AUTH_TOKEN=xxx # Required with LIBSQL_URL
|
||||
|
||||
# NEAR AI (required)
|
||||
NEARAI_SESSION_TOKEN=sess_...
|
||||
@@ -308,7 +320,51 @@ Session tokens have the format `sess_xxx` (37 characters). They are authenticate
|
||||
|
||||
## Database
|
||||
|
||||
Single migration in `migrations/V1__initial.sql`. Tables:
|
||||
IronClaw supports two database backends, selected at compile time via Cargo feature flags and at runtime via the `DATABASE_BACKEND` environment variable.
|
||||
|
||||
**IMPORTANT: All new features that touch persistence MUST support both backends.** Implement the operation as a method on the `Database` trait in `src/db/mod.rs`, then add the implementation in both `src/db/postgres.rs` (delegate to Store/Repository) and `src/db/libsql_backend.rs` (native SQL).
|
||||
|
||||
### Backends
|
||||
|
||||
| Backend | Feature Flag | Default | Use Case |
|
||||
|---------|-------------|---------|----------|
|
||||
| PostgreSQL | `postgres` (default) | Yes | Production, existing deployments |
|
||||
| libSQL/Turso | `libsql` | No | Zero-dependency local mode, edge, Turso cloud |
|
||||
|
||||
```bash
|
||||
# Build with PostgreSQL only (default)
|
||||
cargo build
|
||||
|
||||
# Build with libSQL only
|
||||
cargo build --no-default-features --features libsql
|
||||
|
||||
# Build with both backends available
|
||||
cargo build --features "postgres,libsql"
|
||||
```
|
||||
|
||||
### Database Trait
|
||||
|
||||
The `Database` trait (`src/db/mod.rs`) defines ~60 async methods covering all persistence:
|
||||
- Conversations, messages, metadata
|
||||
- Jobs, actions, LLM calls, estimation snapshots
|
||||
- Sandbox jobs, job events
|
||||
- Routines, routine runs
|
||||
- Tool failures, settings
|
||||
- Workspace: documents, chunks, hybrid search
|
||||
|
||||
Both backends implement this trait. PostgreSQL delegates to the existing `Store` + `Repository`. libSQL implements native SQLite-dialect SQL.
|
||||
|
||||
### Schema
|
||||
|
||||
**PostgreSQL:** `migrations/V1__initial.sql` (351 lines). Uses pgvector for embeddings, tsvector for FTS, PL/pgSQL functions. Managed by `refinery`.
|
||||
|
||||
**libSQL:** `src/db/libsql_migrations.rs` (consolidated schema, ~480 lines). Translates PG types:
|
||||
- `UUID` -> `TEXT`, `TIMESTAMPTZ` -> `TEXT` (ISO-8601), `JSONB` -> `TEXT`
|
||||
- `VECTOR(1536)` -> `F32_BLOB(1536)` with `libsql_vector_idx`
|
||||
- `tsvector`/`ts_rank_cd` -> FTS5 virtual table with sync triggers
|
||||
- PL/pgSQL functions -> SQLite triggers
|
||||
|
||||
**Tables (both backends):**
|
||||
|
||||
**Core:**
|
||||
- `conversations` - Multi-channel conversation tracking
|
||||
@@ -320,12 +376,41 @@ Single migration in `migrations/V1__initial.sql`. Tables:
|
||||
|
||||
**Workspace/Memory:**
|
||||
- `memory_documents` - Flexible path-based files (e.g., "context/vision.md", "daily/2024-01-15.md")
|
||||
- `memory_chunks` - Chunked content with FTS (tsvector) and vector (pgvector) indexes
|
||||
- `memory_chunks` - Chunked content with FTS and vector indexes
|
||||
- `heartbeat_state` - Periodic execution tracking
|
||||
|
||||
Requires pgvector extension: `CREATE EXTENSION IF NOT EXISTS vector;`
|
||||
**Other:**
|
||||
- `routines`, `routine_runs` - Scheduled/reactive execution
|
||||
- `settings` - Per-user key-value settings
|
||||
- `tool_failures` - Self-repair tracking
|
||||
- `secrets`, `wasm_tools`, `tool_capabilities` - Extension infrastructure
|
||||
|
||||
Run migrations: `refinery migrate -c refinery.toml`
|
||||
### Configuration
|
||||
|
||||
```bash
|
||||
# Backend selection (default: postgres)
|
||||
DATABASE_BACKEND=libsql
|
||||
|
||||
# PostgreSQL
|
||||
DATABASE_URL=postgres://user:pass@localhost/ironclaw
|
||||
|
||||
# libSQL (embedded)
|
||||
LIBSQL_PATH=~/.ironclaw/ironclaw.db # Default path
|
||||
|
||||
# libSQL (Turso cloud sync)
|
||||
LIBSQL_URL=libsql://your-db.turso.io
|
||||
LIBSQL_AUTH_TOKEN=your-token # Required when LIBSQL_URL is set
|
||||
```
|
||||
|
||||
### Current Limitations (libSQL backend)
|
||||
|
||||
- **Workspace/memory system** not yet wired through Database trait (requires Store migration)
|
||||
- **Secrets store** not yet available (still requires PostgresSecretsStore)
|
||||
- **Hybrid search** uses FTS5 only (vector search via libsql_vector_idx not yet implemented)
|
||||
- **Settings reload from DB** skipped (Config::from_db requires Store)
|
||||
- No incremental migration versioning (schema is CREATE IF NOT EXISTS, no ALTER TABLE support yet)
|
||||
- **No encryption at rest** -- The local SQLite database file stores conversation content, job data, workspace memory, and other application data in plaintext. Only secrets (API tokens, credentials) are encrypted via AES-256-GCM before storage. Users handling sensitive data should use full-disk encryption (FileVault, LUKS, BitLocker) or consider the PostgreSQL backend with TDE/encrypted storage.
|
||||
- **JSON merge patch vs path-targeted update** -- The libSQL backend uses RFC 7396 JSON Merge Patch (`json_patch`) for metadata updates, while PostgreSQL uses path-targeted `jsonb_set`. Merge patch replaces top-level keys entirely, which may drop nested keys not present in the patch. Callers should avoid relying on partial nested object updates in metadata fields.
|
||||
|
||||
## Safety Layer
|
||||
|
||||
@@ -387,6 +472,7 @@ Key test patterns:
|
||||
- ✅ **Claude Code mode** - Delegate jobs to Claude CLI inside containers
|
||||
- ✅ **Routines system** - Cron, event, webhook, and manual triggers with guardrails
|
||||
- ✅ **Extension management** - Install, auth, activate MCP/WASM extensions via CLI and web UI
|
||||
- ✅ **libSQL/Turso backend** - Database trait abstraction (`src/db/`), feature-gated dual backend support (postgres/libsql), embedded SQLite for zero-dependency local mode
|
||||
|
||||
## Adding a New Tool
|
||||
|
||||
@@ -625,7 +711,7 @@ Four tools for LLM use:
|
||||
|
||||
### Hybrid Search (RRF)
|
||||
|
||||
Combines full-text search (PostgreSQL `ts_rank_cd`) and vector similarity (pgvector cosine) using Reciprocal Rank Fusion:
|
||||
Combines full-text search and vector similarity using Reciprocal Rank Fusion:
|
||||
|
||||
```
|
||||
score(d) = Σ 1/(k + rank(d)) for each method where d appears
|
||||
@@ -633,6 +719,10 @@ score(d) = Σ 1/(k + rank(d)) for each method where d appears
|
||||
|
||||
Default k=60. Results from both methods are combined, with documents appearing in both getting boosted scores.
|
||||
|
||||
**Backend differences:**
|
||||
- **PostgreSQL:** `ts_rank_cd` for FTS, pgvector cosine distance for vectors, full RRF
|
||||
- **libSQL:** FTS5 for keyword search only (vector search via `libsql_vector_idx` not yet wired)
|
||||
|
||||
### Heartbeat System
|
||||
|
||||
Proactive periodic execution (default: 30 minutes):
|
||||
|
||||
Generated
+636
-59
File diff suppressed because it is too large
Load Diff
+25
-8
@@ -28,11 +28,14 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
|
||||
# Database
|
||||
deadpool-postgres = "0.14"
|
||||
tokio-postgres = { version = "0.7", features = ["with-uuid-1", "with-chrono-0_4", "with-serde_json-1"] }
|
||||
postgres-types = { version = "0.2", features = ["with-serde_json-1"] }
|
||||
refinery = { version = "0.8", features = ["tokio-postgres"] }
|
||||
# Database - PostgreSQL (default, feature-gated)
|
||||
deadpool-postgres = { version = "0.14", optional = true }
|
||||
tokio-postgres = { version = "0.7", features = ["with-uuid-1", "with-chrono-0_4", "with-serde_json-1"], optional = true }
|
||||
postgres-types = { version = "0.2", features = ["with-serde_json-1"], optional = true }
|
||||
refinery = { version = "0.8", features = ["tokio-postgres"], optional = true }
|
||||
|
||||
# Database - libSQL/Turso (optional embedded database)
|
||||
libsql = { version = "0.6", optional = true, default-features = false, features = ["core", "replication"] }
|
||||
|
||||
# Error handling
|
||||
thiserror = "2"
|
||||
@@ -48,7 +51,7 @@ dotenvy = "0.15"
|
||||
# Core types
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
rust_decimal = { version = "1", features = ["serde", "serde-with-str", "db-tokio-postgres", "maths"] }
|
||||
rust_decimal = { version = "1", features = ["serde", "serde-with-str", "maths"] }
|
||||
rust_decimal_macros = "1"
|
||||
|
||||
# Async traits
|
||||
@@ -89,7 +92,7 @@ open = "5"
|
||||
|
||||
# Vector embeddings for semantic search
|
||||
# The postgres feature provides ToSql/FromSql for postgres-types (shared by tokio-postgres)
|
||||
pgvector = { version = "0.4", features = ["postgres"] }
|
||||
pgvector = { version = "0.4", features = ["postgres"], optional = true }
|
||||
|
||||
# WASM sandbox for untrusted tool execution
|
||||
wasmtime = { version = "28", features = ["component-model"] }
|
||||
@@ -102,6 +105,7 @@ hkdf = "0.12"
|
||||
sha2 = "0.10"
|
||||
blake3 = "1"
|
||||
rand = "0.8"
|
||||
subtle = "2" # Constant-time comparisons for token validation
|
||||
|
||||
# Multi-provider LLM support
|
||||
rig-core = "0.30"
|
||||
@@ -134,9 +138,22 @@ pretty_assertions = "1"
|
||||
tempfile = "3"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
default = ["postgres"]
|
||||
postgres = [
|
||||
"dep:deadpool-postgres",
|
||||
"dep:tokio-postgres",
|
||||
"dep:postgres-types",
|
||||
"dep:refinery",
|
||||
"dep:pgvector",
|
||||
"rust_decimal/db-tokio-postgres",
|
||||
]
|
||||
libsql = ["dep:libsql"]
|
||||
integration = []
|
||||
|
||||
[[example]]
|
||||
name = "test_heartbeat"
|
||||
required-features = ["postgres"]
|
||||
|
||||
# The profile that 'cargo dist' will build with
|
||||
[profile.dist]
|
||||
inherits = "release"
|
||||
|
||||
+1
-1
@@ -37,7 +37,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O
|
||||
| Session management/routing | ✅ | ✅ | SessionManager exists |
|
||||
| Configuration hot-reload | ✅ | ❌ | |
|
||||
| Network modes (loopback/LAN/remote) | ✅ | 🚧 | HTTP only |
|
||||
| OpenAI-compatible HTTP API | ✅ | ❌ | /v1/chat/completions |
|
||||
| OpenAI-compatible HTTP API | ✅ | ✅ | /v1/chat/completions |
|
||||
| Canvas hosting | ✅ | ❌ | Agent-driven UI |
|
||||
| Gateway lock (PID-based) | ✅ | ❌ | |
|
||||
| launchd/systemd integration | ✅ | ❌ | |
|
||||
|
||||
@@ -856,6 +856,7 @@ fn send_pairing_reply(chat_id: i64, code: &str) -> Result<(), String> {
|
||||
"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage",
|
||||
&headers.to_string(),
|
||||
Some(&payload_bytes),
|
||||
None,
|
||||
);
|
||||
|
||||
match result {
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
# Local development only — do NOT use these credentials in production.
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
ports:
|
||||
- "5432:5432"
|
||||
environment:
|
||||
POSTGRES_DB: ironclaw
|
||||
POSTGRES_USER: ironclaw
|
||||
POSTGRES_PASSWORD: ironclaw # dev-only, change for any non-local deployment
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U ironclaw"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
+40
-6
@@ -19,9 +19,9 @@ use crate::channels::{ChannelManager, IncomingMessage, OutgoingResponse, StatusU
|
||||
use crate::config::{AgentConfig, HeartbeatConfig, RoutineConfig};
|
||||
use crate::context::ContextManager;
|
||||
use crate::context::JobContext;
|
||||
use crate::db::Database;
|
||||
use crate::error::Error;
|
||||
use crate::extensions::ExtensionManager;
|
||||
use crate::history::Store;
|
||||
use crate::llm::{ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult};
|
||||
use crate::safety::SafetyLayer;
|
||||
use crate::tools::ToolRegistry;
|
||||
@@ -59,7 +59,7 @@ enum AgenticLoopResult {
|
||||
///
|
||||
/// Bundles the shared components to reduce argument count.
|
||||
pub struct AgentDeps {
|
||||
pub store: Option<Arc<Store>>,
|
||||
pub store: Option<Arc<dyn Database>>,
|
||||
pub llm: Arc<dyn LlmProvider>,
|
||||
pub safety: Arc<SafetyLayer>,
|
||||
pub tools: Arc<ToolRegistry>,
|
||||
@@ -124,7 +124,7 @@ impl Agent {
|
||||
}
|
||||
|
||||
// Convenience accessors
|
||||
fn store(&self) -> Option<&Arc<Store>> {
|
||||
fn store(&self) -> Option<&Arc<dyn Database>> {
|
||||
self.deps.store.as_ref()
|
||||
}
|
||||
|
||||
@@ -1082,9 +1082,16 @@ impl Agent {
|
||||
m
|
||||
});
|
||||
|
||||
let result = reasoning.respond_with_tools(&context).await?;
|
||||
let output = reasoning.respond_with_tools(&context).await?;
|
||||
|
||||
match result {
|
||||
// Track token usage for budget enforcement
|
||||
tracing::debug!(
|
||||
"LLM call used {} input + {} output tokens",
|
||||
output.usage.input_tokens,
|
||||
output.usage.output_tokens
|
||||
);
|
||||
|
||||
match output.result {
|
||||
RespondResult::Text(text) => {
|
||||
// If no tools have been executed yet, prompt the LLM to use tools
|
||||
// This handles the case where the model explains what it will do
|
||||
@@ -1148,11 +1155,38 @@ impl Agent {
|
||||
if let Some(tool) = self.tools().get(&tc.name).await {
|
||||
if tool.requires_approval() {
|
||||
// Check if auto-approved for this session
|
||||
let is_auto_approved = {
|
||||
let mut is_auto_approved = {
|
||||
let sess = session.lock().await;
|
||||
sess.is_tool_auto_approved(&tc.name)
|
||||
};
|
||||
|
||||
// For shell commands, override auto-approval for
|
||||
// destructive patterns that should always require
|
||||
// explicit per-invocation approval.
|
||||
if is_auto_approved && tc.name == "shell" {
|
||||
if let Some(cmd) = tc
|
||||
.arguments
|
||||
.as_str()
|
||||
.and_then(|s| {
|
||||
serde_json::from_str::<serde_json::Value>(s).ok()
|
||||
})
|
||||
.and_then(|v| {
|
||||
v.get("command")
|
||||
.and_then(|c| c.as_str().map(String::from))
|
||||
})
|
||||
{
|
||||
if crate::tools::builtin::shell::requires_explicit_approval(
|
||||
&cmd,
|
||||
) {
|
||||
tracing::info!(
|
||||
"Shell command '{}' requires explicit approval despite auto-approve",
|
||||
cmd.chars().take(80).collect::<String>()
|
||||
);
|
||||
is_auto_approved = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !is_auto_approved {
|
||||
// Need approval - store pending request and return
|
||||
let pending = PendingApproval {
|
||||
|
||||
+15
-19
@@ -11,6 +11,7 @@
|
||||
//! Full-job routines are delegated to the existing `Scheduler`.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use chrono::Utc;
|
||||
@@ -23,20 +24,20 @@ use crate::agent::routine::{
|
||||
};
|
||||
use crate::channels::{IncomingMessage, OutgoingResponse};
|
||||
use crate::config::RoutineConfig;
|
||||
use crate::history::Store;
|
||||
use crate::db::Database;
|
||||
use crate::llm::{ChatMessage, CompletionRequest, FinishReason, LlmProvider};
|
||||
use crate::workspace::Workspace;
|
||||
|
||||
/// The routine execution engine.
|
||||
pub struct RoutineEngine {
|
||||
config: RoutineConfig,
|
||||
store: Arc<Store>,
|
||||
store: Arc<dyn Database>,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
workspace: Arc<Workspace>,
|
||||
/// Sender for notifications (routed to channel manager).
|
||||
notify_tx: mpsc::Sender<OutgoingResponse>,
|
||||
/// Currently running routine count (across all routines).
|
||||
running_count: Arc<RwLock<usize>>,
|
||||
running_count: Arc<AtomicUsize>,
|
||||
/// Compiled event regex cache: routine_id -> compiled regex.
|
||||
event_cache: Arc<RwLock<Vec<(Uuid, Routine, Regex)>>>,
|
||||
}
|
||||
@@ -44,7 +45,7 @@ pub struct RoutineEngine {
|
||||
impl RoutineEngine {
|
||||
pub fn new(
|
||||
config: RoutineConfig,
|
||||
store: Arc<Store>,
|
||||
store: Arc<dyn Database>,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
workspace: Arc<Workspace>,
|
||||
notify_tx: mpsc::Sender<OutgoingResponse>,
|
||||
@@ -55,7 +56,7 @@ impl RoutineEngine {
|
||||
llm,
|
||||
workspace,
|
||||
notify_tx,
|
||||
running_count: Arc::new(RwLock::new(0)),
|
||||
running_count: Arc::new(AtomicUsize::new(0)),
|
||||
event_cache: Arc::new(RwLock::new(Vec::new())),
|
||||
}
|
||||
}
|
||||
@@ -126,7 +127,7 @@ impl RoutineEngine {
|
||||
}
|
||||
|
||||
// Global capacity check
|
||||
if *self.running_count.read().await >= self.config.max_concurrent_routines {
|
||||
if self.running_count.load(Ordering::Relaxed) >= self.config.max_concurrent_routines {
|
||||
tracing::warn!(routine = %routine.name, "Skipped: global max concurrent reached");
|
||||
continue;
|
||||
}
|
||||
@@ -150,7 +151,7 @@ impl RoutineEngine {
|
||||
};
|
||||
|
||||
for routine in routines {
|
||||
if *self.running_count.read().await >= self.config.max_concurrent_routines {
|
||||
if self.running_count.load(Ordering::Relaxed) >= self.config.max_concurrent_routines {
|
||||
tracing::warn!("Global max concurrent routines reached, skipping remaining");
|
||||
break;
|
||||
}
|
||||
@@ -293,21 +294,18 @@ impl RoutineEngine {
|
||||
|
||||
/// Shared context passed to the execution function.
|
||||
struct EngineContext {
|
||||
store: Arc<Store>,
|
||||
store: Arc<dyn Database>,
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
workspace: Arc<Workspace>,
|
||||
notify_tx: mpsc::Sender<OutgoingResponse>,
|
||||
running_count: Arc<RwLock<usize>>,
|
||||
running_count: Arc<AtomicUsize>,
|
||||
max_lightweight_tokens: u32,
|
||||
}
|
||||
|
||||
/// Execute a routine run. Handles both lightweight and full_job modes.
|
||||
async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun) {
|
||||
// Increment running count
|
||||
{
|
||||
let mut count = ctx.running_count.write().await;
|
||||
*count += 1;
|
||||
}
|
||||
// Increment running count (atomic: survives panics in the execution below)
|
||||
ctx.running_count.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
let result = match &routine.action {
|
||||
RoutineAction::Lightweight {
|
||||
@@ -327,10 +325,7 @@ async fn execute_routine(ctx: EngineContext, routine: Routine, run: RoutineRun)
|
||||
};
|
||||
|
||||
// Decrement running count
|
||||
{
|
||||
let mut count = ctx.running_count.write().await;
|
||||
*count = count.saturating_sub(1);
|
||||
}
|
||||
ctx.running_count.fetch_sub(1, Ordering::Relaxed);
|
||||
|
||||
// Process result
|
||||
let (status, summary, tokens) = match result {
|
||||
@@ -568,7 +563,8 @@ fn truncate(s: &str, max: usize) -> String {
|
||||
if s.len() <= max {
|
||||
s.to_string()
|
||||
} else {
|
||||
format!("{}...", &s[..max])
|
||||
let end = crate::util::floor_char_boundary(s, max);
|
||||
format!("{}...", &s[..end])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+56
-56
@@ -12,8 +12,8 @@ use crate::agent::task::{Task, TaskContext, TaskOutput};
|
||||
use crate::agent::worker::{Worker, WorkerDeps};
|
||||
use crate::config::AgentConfig;
|
||||
use crate::context::{ContextManager, JobContext, JobState};
|
||||
use crate::db::Database;
|
||||
use crate::error::{Error, JobError};
|
||||
use crate::history::Store;
|
||||
use crate::llm::LlmProvider;
|
||||
use crate::safety::SafetyLayer;
|
||||
use crate::tools::ToolRegistry;
|
||||
@@ -48,7 +48,7 @@ pub struct Scheduler {
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
tools: Arc<ToolRegistry>,
|
||||
store: Option<Arc<Store>>,
|
||||
store: Option<Arc<dyn Database>>,
|
||||
/// Running jobs (main LLM-driven jobs).
|
||||
jobs: Arc<RwLock<HashMap<Uuid, ScheduledJob>>>,
|
||||
/// Running sub-tasks (tool executions, background tasks).
|
||||
@@ -63,7 +63,7 @@ impl Scheduler {
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
tools: Arc<ToolRegistry>,
|
||||
store: Option<Arc<Store>>,
|
||||
store: Option<Arc<dyn Database>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
config,
|
||||
@@ -79,63 +79,63 @@ impl Scheduler {
|
||||
|
||||
/// Schedule a job for execution.
|
||||
pub async fn schedule(&self, job_id: Uuid) -> Result<(), JobError> {
|
||||
// Check if already scheduled
|
||||
if self.jobs.read().await.contains_key(&job_id) {
|
||||
return Ok(());
|
||||
}
|
||||
// Hold write lock for the entire check-insert sequence to prevent
|
||||
// TOCTOU races where two concurrent calls both pass the checks.
|
||||
{
|
||||
let mut jobs = self.jobs.write().await;
|
||||
|
||||
// Check capacity
|
||||
let current_count = self.jobs.read().await.len();
|
||||
if current_count >= self.config.max_parallel_jobs {
|
||||
return Err(JobError::MaxJobsExceeded {
|
||||
max: self.config.max_parallel_jobs,
|
||||
});
|
||||
}
|
||||
|
||||
// Transition job to in_progress
|
||||
self.context_manager
|
||||
.update_context(job_id, |ctx| {
|
||||
ctx.transition_to(
|
||||
JobState::InProgress,
|
||||
Some("Scheduled for execution".to_string()),
|
||||
)
|
||||
})
|
||||
.await?
|
||||
.map_err(|s| JobError::ContextError {
|
||||
id: job_id,
|
||||
reason: s,
|
||||
})?;
|
||||
|
||||
// Create worker channel
|
||||
let (tx, rx) = mpsc::channel(16);
|
||||
|
||||
// Create worker with shared dependencies
|
||||
let deps = WorkerDeps {
|
||||
context_manager: self.context_manager.clone(),
|
||||
llm: self.llm.clone(),
|
||||
safety: self.safety.clone(),
|
||||
tools: self.tools.clone(),
|
||||
store: self.store.clone(),
|
||||
timeout: self.config.job_timeout,
|
||||
use_planning: self.config.use_planning,
|
||||
};
|
||||
let worker = Worker::new(job_id, deps);
|
||||
|
||||
// Spawn worker task
|
||||
let handle = tokio::spawn(async move {
|
||||
if let Err(e) = worker.run(rx).await {
|
||||
tracing::error!("Worker for job {} failed: {}", job_id, e);
|
||||
if jobs.contains_key(&job_id) {
|
||||
return Ok(());
|
||||
}
|
||||
});
|
||||
|
||||
// Start the worker
|
||||
let _ = tx.send(WorkerMessage::Start).await;
|
||||
if jobs.len() >= self.config.max_parallel_jobs {
|
||||
return Err(JobError::MaxJobsExceeded {
|
||||
max: self.config.max_parallel_jobs,
|
||||
});
|
||||
}
|
||||
|
||||
// Store the scheduled job
|
||||
self.jobs
|
||||
.write()
|
||||
.await
|
||||
.insert(job_id, ScheduledJob { handle, tx });
|
||||
// Transition job to in_progress
|
||||
self.context_manager
|
||||
.update_context(job_id, |ctx| {
|
||||
ctx.transition_to(
|
||||
JobState::InProgress,
|
||||
Some("Scheduled for execution".to_string()),
|
||||
)
|
||||
})
|
||||
.await?
|
||||
.map_err(|s| JobError::ContextError {
|
||||
id: job_id,
|
||||
reason: s,
|
||||
})?;
|
||||
|
||||
// Create worker channel
|
||||
let (tx, rx) = mpsc::channel(16);
|
||||
|
||||
// Create worker with shared dependencies
|
||||
let deps = WorkerDeps {
|
||||
context_manager: self.context_manager.clone(),
|
||||
llm: self.llm.clone(),
|
||||
safety: self.safety.clone(),
|
||||
tools: self.tools.clone(),
|
||||
store: self.store.clone(),
|
||||
timeout: self.config.job_timeout,
|
||||
use_planning: self.config.use_planning,
|
||||
};
|
||||
let worker = Worker::new(job_id, deps);
|
||||
|
||||
// Spawn worker task
|
||||
let handle = tokio::spawn(async move {
|
||||
if let Err(e) = worker.run(rx).await {
|
||||
tracing::error!("Worker for job {} failed: {}", job_id, e);
|
||||
}
|
||||
});
|
||||
|
||||
// Start the worker
|
||||
let _ = tx.send(WorkerMessage::Start).await;
|
||||
|
||||
// Insert while still holding the write lock
|
||||
jobs.insert(job_id, ScheduledJob { handle, tx });
|
||||
}
|
||||
|
||||
// Cleanup task for this job to avoid capacity leaks
|
||||
let jobs = Arc::clone(&self.jobs);
|
||||
|
||||
@@ -8,8 +8,8 @@ use chrono::{DateTime, Utc};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::context::{ContextManager, JobState};
|
||||
use crate::db::Database;
|
||||
use crate::error::RepairError;
|
||||
use crate::history::Store;
|
||||
use crate::tools::{BuildRequirement, Language, SoftwareBuilder, SoftwareType, ToolRegistry};
|
||||
|
||||
/// A job that has been detected as stuck.
|
||||
@@ -69,7 +69,7 @@ pub struct DefaultSelfRepair {
|
||||
#[allow(dead_code)] // Will be used for time-based stuck detection
|
||||
stuck_threshold: Duration,
|
||||
max_repair_attempts: u32,
|
||||
store: Option<Arc<Store>>,
|
||||
store: Option<Arc<dyn Database>>,
|
||||
builder: Option<Arc<dyn SoftwareBuilder>>,
|
||||
#[allow(dead_code)] // Will be used for tool hot-reload after repair
|
||||
tools: Option<Arc<ToolRegistry>>,
|
||||
@@ -94,7 +94,7 @@ impl DefaultSelfRepair {
|
||||
|
||||
/// Add a Store for tool failure tracking.
|
||||
#[allow(dead_code)] // Public API for configuring repair with persistence
|
||||
pub fn with_store(mut self, store: Arc<Store>) -> Self {
|
||||
pub fn with_store(mut self, store: Arc<dyn Database>) -> Self {
|
||||
self.store = Some(store);
|
||||
self
|
||||
}
|
||||
|
||||
+80
-23
@@ -10,8 +10,8 @@ use uuid::Uuid;
|
||||
use crate::agent::scheduler::WorkerMessage;
|
||||
use crate::agent::task::TaskOutput;
|
||||
use crate::context::{ContextManager, JobState};
|
||||
use crate::db::Database;
|
||||
use crate::error::Error;
|
||||
use crate::history::Store;
|
||||
use crate::llm::{
|
||||
ActionPlan, ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult, ToolSelection,
|
||||
};
|
||||
@@ -28,7 +28,7 @@ pub struct WorkerDeps {
|
||||
pub llm: Arc<dyn LlmProvider>,
|
||||
pub safety: Arc<SafetyLayer>,
|
||||
pub tools: Arc<ToolRegistry>,
|
||||
pub store: Option<Arc<Store>>,
|
||||
pub store: Option<Arc<dyn Database>>,
|
||||
pub timeout: Duration,
|
||||
pub use_planning: bool,
|
||||
}
|
||||
@@ -67,7 +67,7 @@ impl Worker {
|
||||
&self.deps.tools
|
||||
}
|
||||
|
||||
fn store(&self) -> Option<&Arc<Store>> {
|
||||
fn store(&self) -> Option<&Arc<dyn Database>> {
|
||||
self.deps.store.as_ref()
|
||||
}
|
||||
|
||||
@@ -248,16 +248,15 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
||||
|
||||
if selections.is_empty() {
|
||||
// No tools from select_tools, ask LLM directly (may still return tool calls)
|
||||
let respond_result = reasoning.respond_with_tools(reason_ctx).await?;
|
||||
let respond_output = reasoning.respond_with_tools(reason_ctx).await?;
|
||||
|
||||
match respond_result {
|
||||
match respond_output.result {
|
||||
RespondResult::Text(response) => {
|
||||
// Check for completion keywords
|
||||
let response_lower = response.to_lowercase();
|
||||
if response_lower.contains("complete")
|
||||
|| response_lower.contains("finished")
|
||||
|| response_lower.contains("done")
|
||||
{
|
||||
// Check for explicit completion phrases. Use word-boundary
|
||||
// aware checks to avoid false positives like "incomplete",
|
||||
// "not done", or "unfinished". Only the LLM's own response
|
||||
// (not tool output) can trigger this.
|
||||
if crate::util::llm_signals_completion(&response) {
|
||||
self.mark_completed().await?;
|
||||
return Ok(());
|
||||
}
|
||||
@@ -382,7 +381,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
||||
tools: Arc<ToolRegistry>,
|
||||
context_manager: Arc<ContextManager>,
|
||||
safety: Arc<SafetyLayer>,
|
||||
store: Option<Arc<Store>>,
|
||||
store: Option<Arc<dyn Database>>,
|
||||
job_id: Uuid,
|
||||
tool_name: &str,
|
||||
params: &serde_json::Value,
|
||||
@@ -571,12 +570,9 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
||||
wrapped,
|
||||
));
|
||||
|
||||
// Check if job is complete
|
||||
if output.contains("TASK_COMPLETE") || output.contains("JOB_DONE") {
|
||||
self.mark_completed().await?;
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
// Tool output never drives job completion. A malicious tool could
|
||||
// emit "TASK_COMPLETE" to force premature completion. Only the LLM's
|
||||
// own structured response (in execution_loop) can mark a job done.
|
||||
Ok(false)
|
||||
}
|
||||
Err(e) => {
|
||||
@@ -680,11 +676,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."#
|
||||
let response = reasoning.respond(reason_ctx).await?;
|
||||
reason_ctx.messages.push(ChatMessage::assistant(&response));
|
||||
|
||||
let response_lower = response.to_lowercase();
|
||||
if response_lower.contains("complete")
|
||||
|| response_lower.contains("finished")
|
||||
|| response_lower.contains("done")
|
||||
{
|
||||
if crate::util::llm_signals_completion(&response) {
|
||||
self.mark_completed().await?;
|
||||
} else {
|
||||
// Job not complete, could re-plan or fall back to direct selection
|
||||
@@ -779,3 +771,68 @@ impl From<TaskOutput> for Result<String, Error> {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::util::llm_signals_completion;
|
||||
|
||||
#[test]
|
||||
fn test_completion_positive_signals() {
|
||||
assert!(llm_signals_completion("The job is complete."));
|
||||
assert!(llm_signals_completion(
|
||||
"I have completed the task successfully."
|
||||
));
|
||||
assert!(llm_signals_completion("The task is done."));
|
||||
assert!(llm_signals_completion("The task is finished."));
|
||||
assert!(llm_signals_completion(
|
||||
"All steps are complete and verified."
|
||||
));
|
||||
assert!(llm_signals_completion(
|
||||
"I've done all the work. The work is done."
|
||||
));
|
||||
assert!(llm_signals_completion(
|
||||
"Successfully completed the migration."
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_completion_negative_signals_block_false_positives() {
|
||||
// These contain completion keywords but also negation, should NOT trigger.
|
||||
assert!(!llm_signals_completion("The task is not complete yet."));
|
||||
assert!(!llm_signals_completion("This is not done."));
|
||||
assert!(!llm_signals_completion("The work is incomplete."));
|
||||
assert!(!llm_signals_completion(
|
||||
"The migration is not yet finished."
|
||||
));
|
||||
assert!(!llm_signals_completion("The job isn't done yet."));
|
||||
assert!(!llm_signals_completion("This remains unfinished."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_completion_does_not_match_bare_substrings() {
|
||||
// Bare words embedded in other text should NOT trigger completion.
|
||||
assert!(!llm_signals_completion(
|
||||
"I need to complete more work first."
|
||||
));
|
||||
assert!(!llm_signals_completion(
|
||||
"Let me finish the remaining steps."
|
||||
));
|
||||
assert!(!llm_signals_completion(
|
||||
"I'm done analyzing, now let me fix it."
|
||||
));
|
||||
assert!(!llm_signals_completion(
|
||||
"I completed step 1 but step 2 remains."
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_completion_tool_output_injection() {
|
||||
// A malicious tool output echoed by the LLM should not trigger
|
||||
// completion unless it forms a genuine completion phrase.
|
||||
assert!(!llm_signals_completion("TASK_COMPLETE"));
|
||||
assert!(!llm_signals_completion("JOB_DONE"));
|
||||
assert!(!llm_signals_completion(
|
||||
"The tool returned: TASK_COMPLETE signal"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -124,7 +124,7 @@ impl BootstrapConfig {
|
||||
/// If both conditions hold, migrates settings, MCP servers, and session data
|
||||
/// to the database, writes `bootstrap.json`, and renames old files to `.migrated`.
|
||||
pub async fn migrate_disk_to_db(
|
||||
store: &crate::history::Store,
|
||||
store: &dyn crate::db::Database,
|
||||
user_id: &str,
|
||||
) -> Result<(), MigrationError> {
|
||||
let legacy_settings_path = BootstrapConfig::legacy_settings_path();
|
||||
|
||||
@@ -273,6 +273,16 @@ impl near::agent::channel_host::Host for ChannelStoreData {
|
||||
.scan_http_request(&url, &header_vec, body.as_deref())
|
||||
.map_err(|e| format!("Potential secret leak blocked: {}", e))?;
|
||||
|
||||
// Get the max response size from capabilities (default 10MB).
|
||||
let max_response_bytes = self
|
||||
.host_state
|
||||
.capabilities()
|
||||
.tool_capabilities
|
||||
.http
|
||||
.as_ref()
|
||||
.map(|h| h.max_response_bytes)
|
||||
.unwrap_or(10 * 1024 * 1024);
|
||||
|
||||
// Make the HTTP request using blocking I/O
|
||||
// We're already in a spawn_blocking context, so we can use block_on
|
||||
let result = tokio::runtime::Handle::current().block_on(async {
|
||||
@@ -325,11 +335,29 @@ impl near::agent::channel_host::Host for ChannelStoreData {
|
||||
})
|
||||
.collect();
|
||||
let headers_json = serde_json::to_string(&response_headers).unwrap_or_default();
|
||||
|
||||
// Enforce max response body size to prevent memory exhaustion.
|
||||
let max_response = max_response_bytes;
|
||||
if let Some(cl) = response.content_length() {
|
||||
if cl as usize > max_response {
|
||||
return Err(format!(
|
||||
"Response body too large: {} bytes exceeds limit of {} bytes",
|
||||
cl, max_response
|
||||
));
|
||||
}
|
||||
}
|
||||
let body = response
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| format!("Failed to read response body: {}", e))?
|
||||
.to_vec();
|
||||
.map_err(|e| format!("Failed to read response body: {}", e))?;
|
||||
if body.len() > max_response {
|
||||
return Err(format!(
|
||||
"Response body too large: {} bytes exceeds limit of {} bytes",
|
||||
body.len(),
|
||||
max_response
|
||||
));
|
||||
}
|
||||
let body = body.to_vec();
|
||||
|
||||
tracing::info!(
|
||||
status = status,
|
||||
|
||||
@@ -6,6 +6,7 @@ use axum::{
|
||||
middleware::Next,
|
||||
response::{IntoResponse, Response},
|
||||
};
|
||||
use subtle::ConstantTimeEq;
|
||||
|
||||
/// Shared auth state injected via axum middleware state.
|
||||
#[derive(Clone)]
|
||||
@@ -23,22 +24,22 @@ pub async fn auth_middleware(
|
||||
request: Request,
|
||||
next: Next,
|
||||
) -> Response {
|
||||
// Try Authorization header first
|
||||
// Try Authorization header first (constant-time comparison)
|
||||
if let Some(auth_header) = headers.get("authorization") {
|
||||
if let Ok(value) = auth_header.to_str() {
|
||||
if let Some(token) = value.strip_prefix("Bearer ") {
|
||||
if token == auth.token {
|
||||
if bool::from(token.as_bytes().ct_eq(auth.token.as_bytes())) {
|
||||
return next.run(request).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to query parameter (for SSE EventSource)
|
||||
// Fall back to query parameter for SSE EventSource (constant-time comparison)
|
||||
if let Some(query) = request.uri().query() {
|
||||
for pair in query.split('&') {
|
||||
if let Some(token) = pair.strip_prefix("token=") {
|
||||
if token == auth.token {
|
||||
if bool::from(token.as_bytes().ct_eq(auth.token.as_bytes())) {
|
||||
return next.run(request).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,8 @@ use tokio::sync::broadcast;
|
||||
use tracing::field::{Field, Visit};
|
||||
use tracing_subscriber::Layer;
|
||||
|
||||
use crate::safety::LeakDetector;
|
||||
|
||||
/// Maximum number of recent log entries kept for late-joining SSE subscribers.
|
||||
const HISTORY_CAP: usize = 500;
|
||||
|
||||
@@ -46,6 +48,8 @@ pub struct LogEntry {
|
||||
pub struct LogBroadcaster {
|
||||
tx: broadcast::Sender<LogEntry>,
|
||||
recent: Mutex<VecDeque<LogEntry>>,
|
||||
/// Scrubs secrets from log messages before broadcasting to SSE clients.
|
||||
leak_detector: LeakDetector,
|
||||
}
|
||||
|
||||
impl LogBroadcaster {
|
||||
@@ -54,10 +58,19 @@ impl LogBroadcaster {
|
||||
Self {
|
||||
tx,
|
||||
recent: Mutex::new(VecDeque::with_capacity(HISTORY_CAP)),
|
||||
leak_detector: LeakDetector::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn send(&self, entry: LogEntry) {
|
||||
pub fn send(&self, mut entry: LogEntry) {
|
||||
// Scrub secrets from the message before it reaches any subscriber.
|
||||
// This is defense-in-depth: even if code elsewhere accidentally logs
|
||||
// a secret, it won't be broadcast to SSE clients.
|
||||
entry.message = self
|
||||
.leak_detector
|
||||
.scan_and_clean(&entry.message)
|
||||
.unwrap_or_else(|_| "[log message redacted: contained blocked secret]".to_string());
|
||||
|
||||
// Stash in ring buffer (for late joiners)
|
||||
if let Ok(mut buf) = self.recent.lock() {
|
||||
if buf.len() >= HISTORY_CAP {
|
||||
@@ -145,6 +158,9 @@ impl Visit for MessageVisitor {
|
||||
///
|
||||
/// Only forwards DEBUG and above. Attach to the tracing subscriber
|
||||
/// alongside the existing fmt layer.
|
||||
///
|
||||
/// Log messages are scrubbed through `LeakDetector` in `LogBroadcaster::send()`
|
||||
/// (the single funnel point for all log output, including late-joiner history).
|
||||
pub struct WebLogLayer {
|
||||
broadcaster: Arc<LogBroadcaster>,
|
||||
}
|
||||
@@ -178,6 +194,7 @@ impl<S: tracing::Subscriber> Layer<S> for WebLogLayer {
|
||||
timestamp: chrono::Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true),
|
||||
};
|
||||
|
||||
// LeakDetector scrubbing happens inside broadcaster.send()
|
||||
self.broadcaster.send(entry);
|
||||
}
|
||||
}
|
||||
@@ -313,4 +330,29 @@ mod tests {
|
||||
let v = MessageVisitor::new();
|
||||
assert_eq!(v.finish(), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_broadcaster_has_leak_detector() {
|
||||
let broadcaster = LogBroadcaster::new();
|
||||
// Verify the leak detector is initialized with default patterns
|
||||
assert!(broadcaster.leak_detector.pattern_count() > 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_leak_detector_scrubs_api_key_in_log() {
|
||||
let detector = crate::safety::LeakDetector::new();
|
||||
let msg = "Connecting with token sk-proj-test1234567890abcdefghij";
|
||||
let result = detector.scan_and_clean(msg);
|
||||
// Should be blocked (OpenAI key pattern)
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_leak_detector_passes_clean_log() {
|
||||
let detector = crate::safety::LeakDetector::new();
|
||||
let msg = "Request completed status=200 url=https://api.example.com/data";
|
||||
let result = detector.scan_and_clean(msg);
|
||||
assert!(result.is_ok());
|
||||
assert_eq!(result.unwrap(), msg);
|
||||
}
|
||||
}
|
||||
|
||||
+13
-2
@@ -16,6 +16,7 @@
|
||||
|
||||
pub mod auth;
|
||||
pub mod log_layer;
|
||||
pub mod openai_compat;
|
||||
pub mod server;
|
||||
pub mod sse;
|
||||
pub mod types;
|
||||
@@ -31,9 +32,9 @@ use tokio_stream::wrappers::ReceiverStream;
|
||||
use crate::agent::SessionManager;
|
||||
use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate};
|
||||
use crate::config::GatewayConfig;
|
||||
use crate::db::Database;
|
||||
use crate::error::ChannelError;
|
||||
use crate::extensions::ExtensionManager;
|
||||
use crate::history::Store;
|
||||
use crate::orchestrator::job_manager::ContainerJobManager;
|
||||
use crate::tools::ToolRegistry;
|
||||
use crate::workspace::Workspace;
|
||||
@@ -81,6 +82,8 @@ impl GatewayChannel {
|
||||
user_id: config.user_id.clone(),
|
||||
shutdown_tx: tokio::sync::RwLock::new(None),
|
||||
ws_tracker: Some(Arc::new(ws::WsConnectionTracker::new())),
|
||||
llm_provider: None,
|
||||
chat_rate_limiter: server::RateLimiter::new(30, 60),
|
||||
});
|
||||
|
||||
Self {
|
||||
@@ -106,6 +109,8 @@ impl GatewayChannel {
|
||||
user_id: self.state.user_id.clone(),
|
||||
shutdown_tx: tokio::sync::RwLock::new(None),
|
||||
ws_tracker: self.state.ws_tracker.clone(),
|
||||
llm_provider: self.state.llm_provider.clone(),
|
||||
chat_rate_limiter: server::RateLimiter::new(30, 60),
|
||||
};
|
||||
mutate(&mut new_state);
|
||||
self.state = Arc::new(new_state);
|
||||
@@ -142,7 +147,7 @@ impl GatewayChannel {
|
||||
}
|
||||
|
||||
/// Inject the database store for sandbox job persistence.
|
||||
pub fn with_store(mut self, store: Arc<Store>) -> Self {
|
||||
pub fn with_store(mut self, store: Arc<dyn Database>) -> Self {
|
||||
self.rebuild_state(|s| s.store = Some(store));
|
||||
self
|
||||
}
|
||||
@@ -169,6 +174,12 @@ impl GatewayChannel {
|
||||
self
|
||||
}
|
||||
|
||||
/// Inject the LLM provider for OpenAI-compatible API proxy.
|
||||
pub fn with_llm_provider(mut self, llm: Arc<dyn crate::llm::LlmProvider>) -> Self {
|
||||
self.rebuild_state(|s| s.llm_provider = Some(llm));
|
||||
self
|
||||
}
|
||||
|
||||
/// Get the auth token (for printing to console on startup).
|
||||
pub fn auth_token(&self) -> &str {
|
||||
&self.auth_token
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+234
-19
@@ -5,10 +5,11 @@
|
||||
use std::convert::Infallible;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::{Path, Query, State, WebSocketUpgrade},
|
||||
extract::{DefaultBodyLimit, Path, Query, State, WebSocketUpgrade},
|
||||
http::{StatusCode, header},
|
||||
middleware,
|
||||
response::{
|
||||
@@ -20,6 +21,7 @@ use axum::{
|
||||
use serde::Deserialize;
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use tokio_stream::StreamExt;
|
||||
use tower_http::cors::{AllowHeaders, CorsLayer};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::agent::SessionManager;
|
||||
@@ -28,8 +30,8 @@ use crate::channels::web::auth::{AuthState, auth_middleware};
|
||||
use crate::channels::web::log_layer::LogBroadcaster;
|
||||
use crate::channels::web::sse::SseManager;
|
||||
use crate::channels::web::types::*;
|
||||
use crate::db::Database;
|
||||
use crate::extensions::ExtensionManager;
|
||||
use crate::history::Store;
|
||||
use crate::orchestrator::job_manager::ContainerJobManager;
|
||||
use crate::tools::ToolRegistry;
|
||||
use crate::workspace::Workspace;
|
||||
@@ -44,6 +46,69 @@ pub type PromptQueue = Arc<
|
||||
>,
|
||||
>;
|
||||
|
||||
/// Simple sliding-window rate limiter.
|
||||
///
|
||||
/// Tracks the number of requests in the current window. Resets when the window expires.
|
||||
/// Not per-IP (since this is a single-user gateway with auth), but prevents flooding.
|
||||
pub struct RateLimiter {
|
||||
/// Requests remaining in the current window.
|
||||
remaining: AtomicU64,
|
||||
/// Epoch second when the current window started.
|
||||
window_start: AtomicU64,
|
||||
/// Maximum requests per window.
|
||||
max_requests: u64,
|
||||
/// Window duration in seconds.
|
||||
window_secs: u64,
|
||||
}
|
||||
|
||||
impl RateLimiter {
|
||||
pub fn new(max_requests: u64, window_secs: u64) -> Self {
|
||||
Self {
|
||||
remaining: AtomicU64::new(max_requests),
|
||||
window_start: AtomicU64::new(
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs(),
|
||||
),
|
||||
max_requests,
|
||||
window_secs,
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to consume one request. Returns `true` if allowed, `false` if rate limited.
|
||||
pub fn check(&self) -> bool {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs();
|
||||
|
||||
let window = self.window_start.load(Ordering::Relaxed);
|
||||
if now.saturating_sub(window) >= self.window_secs {
|
||||
// Window expired, reset
|
||||
self.window_start.store(now, Ordering::Relaxed);
|
||||
self.remaining
|
||||
.store(self.max_requests - 1, Ordering::Relaxed);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Try to decrement remaining
|
||||
loop {
|
||||
let current = self.remaining.load(Ordering::Relaxed);
|
||||
if current == 0 {
|
||||
return false;
|
||||
}
|
||||
if self
|
||||
.remaining
|
||||
.compare_exchange_weak(current, current - 1, Ordering::Relaxed, Ordering::Relaxed)
|
||||
.is_ok()
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared state for all gateway handlers.
|
||||
pub struct GatewayState {
|
||||
/// Channel to send messages to the agent loop.
|
||||
@@ -61,7 +126,7 @@ pub struct GatewayState {
|
||||
/// Tool registry for listing registered tools.
|
||||
pub tool_registry: Option<Arc<ToolRegistry>>,
|
||||
/// Database store for sandbox job persistence.
|
||||
pub store: Option<Arc<Store>>,
|
||||
pub store: Option<Arc<dyn Database>>,
|
||||
/// Container job manager for sandbox operations.
|
||||
pub job_manager: Option<Arc<ContainerJobManager>>,
|
||||
/// Prompt queue for Claude Code follow-up prompts.
|
||||
@@ -72,6 +137,10 @@ pub struct GatewayState {
|
||||
pub shutdown_tx: tokio::sync::RwLock<Option<oneshot::Sender<()>>>,
|
||||
/// WebSocket connection tracker.
|
||||
pub ws_tracker: Option<Arc<crate::channels::web::ws::WsConnectionTracker>>,
|
||||
/// LLM provider for OpenAI-compatible API proxy.
|
||||
pub llm_provider: Option<Arc<dyn crate::llm::LlmProvider>>,
|
||||
/// Rate limiter for chat endpoints (30 messages per 60 seconds).
|
||||
pub chat_rate_limiter: RateLimiter,
|
||||
}
|
||||
|
||||
/// Start the gateway HTTP server.
|
||||
@@ -168,7 +237,16 @@ pub async fn start_server(
|
||||
)
|
||||
// Gateway control plane
|
||||
.route("/api/gateway/status", get(gateway_status_handler))
|
||||
.route_layer(middleware::from_fn_with_state(auth_state, auth_middleware));
|
||||
// OpenAI-compatible API
|
||||
.route(
|
||||
"/v1/chat/completions",
|
||||
post(super::openai_compat::chat_completions_handler),
|
||||
)
|
||||
.route("/v1/models", get(super::openai_compat::models_handler))
|
||||
.route_layer(middleware::from_fn_with_state(
|
||||
auth_state.clone(),
|
||||
auth_middleware,
|
||||
));
|
||||
|
||||
// Static file routes (no auth, served from embedded strings)
|
||||
let statics = Router::new()
|
||||
@@ -176,19 +254,46 @@ pub async fn start_server(
|
||||
.route("/style.css", get(css_handler))
|
||||
.route("/app.js", get(js_handler));
|
||||
|
||||
// Project file serving (no auth, local browsing of sandbox outputs).
|
||||
// The trailing-slash route serves index.html; the bare route redirects so
|
||||
// relative paths in the HTML (e.g. href="style.css") resolve correctly.
|
||||
// Project file serving (behind auth to prevent unauthorized file access).
|
||||
let projects = Router::new()
|
||||
.route("/projects/{project_id}", get(project_redirect_handler))
|
||||
.route("/projects/{project_id}/", get(project_index_handler))
|
||||
.route("/projects/{project_id}/{*path}", get(project_file_handler));
|
||||
.route("/projects/{project_id}/{*path}", get(project_file_handler))
|
||||
.route_layer(middleware::from_fn_with_state(
|
||||
auth_state.clone(),
|
||||
auth_middleware,
|
||||
));
|
||||
|
||||
// CORS: restrict to same-origin by default. Only localhost/127.0.0.1
|
||||
// origins are allowed, since the gateway is a local-first service.
|
||||
let cors = CorsLayer::new()
|
||||
.allow_origin([
|
||||
format!("http://{}:{}", addr.ip(), addr.port())
|
||||
.parse()
|
||||
.expect("valid origin"),
|
||||
format!("http://localhost:{}", addr.port())
|
||||
.parse()
|
||||
.expect("valid origin"),
|
||||
])
|
||||
.allow_methods([
|
||||
axum::http::Method::GET,
|
||||
axum::http::Method::POST,
|
||||
axum::http::Method::PUT,
|
||||
axum::http::Method::DELETE,
|
||||
])
|
||||
.allow_headers(AllowHeaders::list([
|
||||
header::CONTENT_TYPE,
|
||||
header::AUTHORIZATION,
|
||||
]))
|
||||
.allow_credentials(true);
|
||||
|
||||
let app = Router::new()
|
||||
.merge(public)
|
||||
.merge(statics)
|
||||
.merge(projects)
|
||||
.merge(protected)
|
||||
.layer(cors)
|
||||
.layer(DefaultBodyLimit::max(1024 * 1024)) // 1 MB max request body
|
||||
.with_state(state.clone());
|
||||
|
||||
let (shutdown_tx, shutdown_rx) = oneshot::channel();
|
||||
@@ -244,6 +349,13 @@ async fn chat_send_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
Json(req): Json<SendMessageRequest>,
|
||||
) -> Result<(StatusCode, Json<SendMessageResponse>), (StatusCode, String)> {
|
||||
if !state.chat_rate_limiter.check() {
|
||||
return Err((
|
||||
StatusCode::TOO_MANY_REQUESTS,
|
||||
"Rate limit exceeded. Try again shortly.".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let mut msg = IncomingMessage::new("gateway", &state.user_id, &req.content);
|
||||
|
||||
if let Some(ref thread_id) = req.thread_id {
|
||||
@@ -421,16 +533,50 @@ pub async fn clear_auth_mode(state: &GatewayState) {
|
||||
}
|
||||
}
|
||||
|
||||
async fn chat_events_handler(State(state): State<Arc<GatewayState>>) -> impl IntoResponse {
|
||||
// subscribe() returns Sse<impl Stream + 'static + use<>> so no lifetime issues
|
||||
state.sse.subscribe()
|
||||
async fn chat_events_handler(
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
) -> Result<impl IntoResponse, (StatusCode, String)> {
|
||||
state.sse.subscribe().ok_or((
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"Too many connections".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
async fn chat_ws_handler(
|
||||
headers: axum::http::HeaderMap,
|
||||
ws: WebSocketUpgrade,
|
||||
State(state): State<Arc<GatewayState>>,
|
||||
) -> impl IntoResponse {
|
||||
ws.on_upgrade(move |socket| crate::channels::web::ws::handle_ws_connection(socket, state))
|
||||
) -> Result<impl IntoResponse, (StatusCode, String)> {
|
||||
// Validate Origin header to prevent cross-site WebSocket hijacking.
|
||||
// Require the header outright; browsers always send it for WS upgrades,
|
||||
// so a missing Origin means a non-browser client trying to bypass the check.
|
||||
let origin = headers
|
||||
.get("origin")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.ok_or_else(|| {
|
||||
(
|
||||
StatusCode::FORBIDDEN,
|
||||
"WebSocket Origin header required".to_string(),
|
||||
)
|
||||
})?;
|
||||
|
||||
// Extract the host from the origin and compare exactly, so that
|
||||
// crafted origins like "http://localhost.evil.com" are rejected.
|
||||
// Origin format is "scheme://host[:port]".
|
||||
let host = origin
|
||||
.strip_prefix("http://")
|
||||
.or_else(|| origin.strip_prefix("https://"))
|
||||
.and_then(|rest| rest.split(':').next()?.split('/').next())
|
||||
.unwrap_or("");
|
||||
|
||||
let is_local = matches!(host, "localhost" | "127.0.0.1" | "[::1]");
|
||||
if !is_local {
|
||||
return Err((
|
||||
StatusCode::FORBIDDEN,
|
||||
"WebSocket origin not allowed".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(ws.on_upgrade(move |socket| crate::channels::web::ws::handle_ws_connection(socket, state)))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -477,6 +623,21 @@ async fn chat_history_handler(
|
||||
.ok_or((StatusCode::NOT_FOUND, "No active thread".to_string()))?
|
||||
};
|
||||
|
||||
// Verify the thread belongs to the authenticated user before returning any data.
|
||||
// In-memory threads are already scoped by user via session_manager, but DB
|
||||
// lookups could expose another user's conversation if the UUID is guessed.
|
||||
if query.thread_id.is_some() {
|
||||
if let Some(ref store) = state.store {
|
||||
let owned = store
|
||||
.conversation_belongs_to_user(thread_id, &state.user_id)
|
||||
.await
|
||||
.unwrap_or(false);
|
||||
if !owned && !sess.threads.contains_key(&thread_id) {
|
||||
return Err((StatusCode::NOT_FOUND, "Thread not found".to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// For paginated requests (before cursor set), always go to DB
|
||||
if before_cursor.is_some() {
|
||||
if let Some(ref store) = state.store {
|
||||
@@ -901,14 +1062,16 @@ async fn jobs_list_handler(
|
||||
"Database not available".to_string(),
|
||||
))?;
|
||||
|
||||
// Fetch sandbox jobs from the DB.
|
||||
// Fetch sandbox jobs scoped to the authenticated user.
|
||||
let sandbox_jobs = store
|
||||
.list_sandbox_jobs()
|
||||
.list_sandbox_jobs_for_user(&state.user_id)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||
|
||||
// Scope jobs to the authenticated user.
|
||||
let mut jobs: Vec<JobInfo> = sandbox_jobs
|
||||
.iter()
|
||||
.filter(|j| j.user_id == state.user_id)
|
||||
.map(|j| {
|
||||
let ui_state = match j.status.as_str() {
|
||||
"creating" => "pending",
|
||||
@@ -941,7 +1104,7 @@ async fn jobs_summary_handler(
|
||||
))?;
|
||||
|
||||
let s = store
|
||||
.sandbox_job_summary()
|
||||
.sandbox_job_summary_for_user(&state.user_id)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
||||
|
||||
@@ -962,9 +1125,12 @@ async fn jobs_detail_handler(
|
||||
let job_id = Uuid::parse_str(&id)
|
||||
.map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?;
|
||||
|
||||
// Try sandbox job from DB first.
|
||||
// Try sandbox job from DB first, scoped to the authenticated user.
|
||||
if let Some(ref store) = state.store {
|
||||
if let Ok(Some(job)) = store.get_sandbox_job(job_id).await {
|
||||
if job.user_id != state.user_id {
|
||||
return Err((StatusCode::NOT_FOUND, "Job not found".to_string()));
|
||||
}
|
||||
let browse_id = std::path::Path::new(&job.project_dir)
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy().to_string())
|
||||
@@ -1031,13 +1197,18 @@ async fn jobs_cancel_handler(
|
||||
let job_id = Uuid::parse_str(&id)
|
||||
.map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?;
|
||||
|
||||
// Try sandbox job cancellation.
|
||||
// Try sandbox job cancellation, scoped to the authenticated user.
|
||||
if let Some(ref store) = state.store {
|
||||
if let Ok(Some(job)) = store.get_sandbox_job(job_id).await {
|
||||
if job.user_id != state.user_id {
|
||||
return Err((StatusCode::NOT_FOUND, "Job not found".to_string()));
|
||||
}
|
||||
if job.status == "running" || job.status == "creating" {
|
||||
// Stop the container if we have a job manager.
|
||||
if let Some(ref jm) = state.job_manager {
|
||||
let _ = jm.stop_job(job_id).await;
|
||||
if let Err(e) = jm.stop_job(job_id).await {
|
||||
tracing::warn!(job_id = %job_id, error = %e, "Failed to stop container during cancellation");
|
||||
}
|
||||
}
|
||||
store
|
||||
.update_sandbox_job_status(
|
||||
@@ -1083,6 +1254,11 @@ async fn jobs_restart_handler(
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?
|
||||
.ok_or((StatusCode::NOT_FOUND, "Job not found".to_string()))?;
|
||||
|
||||
// Scope to the authenticated user.
|
||||
if old_job.user_id != state.user_id {
|
||||
return Err((StatusCode::NOT_FOUND, "Job not found".to_string()));
|
||||
}
|
||||
|
||||
if old_job.status != "interrupted" && old_job.status != "failed" {
|
||||
return Err((
|
||||
StatusCode::CONFLICT,
|
||||
@@ -1157,6 +1333,17 @@ async fn jobs_prompt_handler(
|
||||
.parse()
|
||||
.map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?;
|
||||
|
||||
// Verify user owns this job.
|
||||
if let Some(ref store) = state.store {
|
||||
if !store
|
||||
.sandbox_job_belongs_to_user(job_id, &state.user_id)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return Err((StatusCode::NOT_FOUND, "Job not found".to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
let content = body
|
||||
.get("content")
|
||||
.and_then(|v| v.as_str())
|
||||
@@ -1195,6 +1382,15 @@ async fn jobs_events_handler(
|
||||
.parse()
|
||||
.map_err(|_| (StatusCode::BAD_REQUEST, "Invalid job ID".to_string()))?;
|
||||
|
||||
// Verify user owns this job.
|
||||
if !store
|
||||
.sandbox_job_belongs_to_user(job_id, &state.user_id)
|
||||
.await
|
||||
.unwrap_or(false)
|
||||
{
|
||||
return Err((StatusCode::NOT_FOUND, "Job not found".to_string()));
|
||||
}
|
||||
|
||||
let events = store
|
||||
.list_job_events(job_id)
|
||||
.await
|
||||
@@ -1244,6 +1440,11 @@ async fn job_files_list_handler(
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?
|
||||
.ok_or((StatusCode::NOT_FOUND, "Job not found".to_string()))?;
|
||||
|
||||
// Verify user owns this job.
|
||||
if job.user_id != state.user_id {
|
||||
return Err((StatusCode::NOT_FOUND, "Job not found".to_string()));
|
||||
}
|
||||
|
||||
let base = std::path::PathBuf::from(&job.project_dir);
|
||||
let rel_path = query.path.as_deref().unwrap_or("");
|
||||
let target = base.join(rel_path);
|
||||
@@ -1307,6 +1508,11 @@ async fn job_files_read_handler(
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?
|
||||
.ok_or((StatusCode::NOT_FOUND, "Job not found".to_string()))?;
|
||||
|
||||
// Verify user owns this job.
|
||||
if job.user_id != state.user_id {
|
||||
return Err((StatusCode::NOT_FOUND, "Job not found".to_string()));
|
||||
}
|
||||
|
||||
let path = query.path.as_deref().ok_or((
|
||||
StatusCode::BAD_REQUEST,
|
||||
"path parameter required".to_string(),
|
||||
@@ -1525,6 +1731,15 @@ async fn project_file_handler(
|
||||
/// Shared logic: resolve the file inside `~/.ironclaw/projects/{project_id}/`,
|
||||
/// guard against path traversal, and stream the content with the right MIME type.
|
||||
async fn serve_project_file(project_id: &str, path: &str) -> axum::response::Response {
|
||||
// Reject project_id values that could escape the projects directory.
|
||||
if project_id.contains('/')
|
||||
|| project_id.contains('\\')
|
||||
|| project_id.contains("..")
|
||||
|| project_id.is_empty()
|
||||
{
|
||||
return (StatusCode::BAD_REQUEST, "Invalid project ID").into_response();
|
||||
}
|
||||
|
||||
let base = dirs::home_dir()
|
||||
.unwrap_or_else(|| std::path::PathBuf::from("."))
|
||||
.join(".ironclaw")
|
||||
|
||||
+59
-12
@@ -13,10 +13,15 @@ use tokio_stream::wrappers::BroadcastStream;
|
||||
|
||||
use crate::channels::web::types::SseEvent;
|
||||
|
||||
/// Maximum number of concurrent SSE/WebSocket connections.
|
||||
/// Prevents resource exhaustion from connection flooding.
|
||||
const MAX_CONNECTIONS: u64 = 100;
|
||||
|
||||
/// Manages SSE broadcast to all connected browser tabs.
|
||||
pub struct SseManager {
|
||||
tx: broadcast::Sender<SseEvent>,
|
||||
connection_count: Arc<AtomicU64>,
|
||||
max_connections: u64,
|
||||
}
|
||||
|
||||
impl SseManager {
|
||||
@@ -27,6 +32,7 @@ impl SseManager {
|
||||
Self {
|
||||
tx,
|
||||
connection_count: Arc::new(AtomicU64::new(0)),
|
||||
max_connections: MAX_CONNECTIONS,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,25 +51,50 @@ impl SseManager {
|
||||
///
|
||||
/// Returns a stream of `SseEvent` values and increments/decrements the
|
||||
/// connection counter on creation/drop, just like `subscribe()` does for SSE.
|
||||
pub fn subscribe_raw(&self) -> impl Stream<Item = SseEvent> + Send + 'static + use<> {
|
||||
///
|
||||
/// Returns `None` if the maximum connection limit has been reached.
|
||||
pub fn subscribe_raw(&self) -> Option<impl Stream<Item = SseEvent> + Send + 'static + use<>> {
|
||||
// Atomically increment only if below the limit. This prevents
|
||||
// concurrent callers from overshooting max_connections.
|
||||
let counter = Arc::clone(&self.connection_count);
|
||||
counter.fetch_add(1, Ordering::Relaxed);
|
||||
let max = self.max_connections;
|
||||
counter
|
||||
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
|
||||
if current < max {
|
||||
Some(current + 1)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.ok()?;
|
||||
let rx = self.tx.subscribe();
|
||||
|
||||
let stream = BroadcastStream::new(rx).filter_map(|result| result.ok());
|
||||
|
||||
CountedStream {
|
||||
Some(CountedStream {
|
||||
inner: stream,
|
||||
counter,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Create a new SSE stream for a client connection.
|
||||
///
|
||||
/// Returns `None` if the maximum connection limit has been reached.
|
||||
pub fn subscribe(
|
||||
&self,
|
||||
) -> Sse<impl Stream<Item = Result<Event, Infallible>> + Send + 'static + use<>> {
|
||||
) -> Option<Sse<impl Stream<Item = Result<Event, Infallible>> + Send + 'static + use<>>> {
|
||||
// Atomically increment only if below the limit.
|
||||
let counter = Arc::clone(&self.connection_count);
|
||||
counter.fetch_add(1, Ordering::Relaxed);
|
||||
let max = self.max_connections;
|
||||
counter
|
||||
.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
|
||||
if current < max {
|
||||
Some(current + 1)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.ok()?;
|
||||
let rx = self.tx.subscribe();
|
||||
|
||||
let stream = BroadcastStream::new(rx)
|
||||
@@ -99,8 +130,10 @@ impl SseManager {
|
||||
counter,
|
||||
};
|
||||
|
||||
Sse::new(counted_stream)
|
||||
.keep_alive(KeepAlive::new().interval(Duration::from_secs(30)).text(""))
|
||||
Some(
|
||||
Sse::new(counted_stream)
|
||||
.keep_alive(KeepAlive::new().interval(Duration::from_secs(30)).text("")),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,7 +208,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn test_subscribe_raw_receives_events() {
|
||||
let manager = SseManager::new();
|
||||
let mut stream = Box::pin(manager.subscribe_raw());
|
||||
let mut stream = Box::pin(manager.subscribe_raw().expect("should subscribe"));
|
||||
|
||||
assert_eq!(manager.connection_count(), 1);
|
||||
|
||||
@@ -195,7 +228,7 @@ mod tests {
|
||||
async fn test_subscribe_raw_decrements_on_drop() {
|
||||
let manager = SseManager::new();
|
||||
{
|
||||
let _stream = Box::pin(manager.subscribe_raw());
|
||||
let _stream = Box::pin(manager.subscribe_raw().expect("should subscribe"));
|
||||
assert_eq!(manager.connection_count(), 1);
|
||||
}
|
||||
// Stream dropped, counter should decrement
|
||||
@@ -205,8 +238,8 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn test_subscribe_raw_multiple_subscribers() {
|
||||
let manager = SseManager::new();
|
||||
let mut s1 = Box::pin(manager.subscribe_raw());
|
||||
let mut s2 = Box::pin(manager.subscribe_raw());
|
||||
let mut s1 = Box::pin(manager.subscribe_raw().expect("should subscribe"));
|
||||
let mut s2 = Box::pin(manager.subscribe_raw().expect("should subscribe"));
|
||||
assert_eq!(manager.connection_count(), 2);
|
||||
|
||||
manager.broadcast(SseEvent::Heartbeat);
|
||||
@@ -221,4 +254,18 @@ mod tests {
|
||||
drop(s2);
|
||||
assert_eq!(manager.connection_count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_subscribe_raw_rejects_over_limit() {
|
||||
let mut manager = SseManager::new();
|
||||
manager.max_connections = 2; // Low limit for testing
|
||||
|
||||
let _s1 = Box::pin(manager.subscribe_raw().expect("first should succeed"));
|
||||
let _s2 = Box::pin(manager.subscribe_raw().expect("second should succeed"));
|
||||
assert_eq!(manager.connection_count(), 2);
|
||||
|
||||
// Third should be rejected
|
||||
assert!(manager.subscribe_raw().is_none());
|
||||
assert!(manager.subscribe().is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -281,6 +281,8 @@ function sendApprovalAction(requestId, action) {
|
||||
function renderMarkdown(text) {
|
||||
if (typeof marked !== 'undefined') {
|
||||
let html = marked.parse(text);
|
||||
// Sanitize HTML output to prevent XSS from tool output or LLM responses.
|
||||
html = sanitizeRenderedHtml(html);
|
||||
// Inject copy buttons into <pre> blocks
|
||||
html = html.replace(/<pre>/g, '<pre class="code-block-wrapper"><button class="copy-btn" onclick="copyCodeBlock(this)">Copy</button>');
|
||||
return html;
|
||||
@@ -288,6 +290,28 @@ function renderMarkdown(text) {
|
||||
return escapeHtml(text);
|
||||
}
|
||||
|
||||
// Strip dangerous HTML elements and attributes from rendered markdown.
|
||||
// This prevents XSS from tool output or prompt injection in LLM responses.
|
||||
function sanitizeRenderedHtml(html) {
|
||||
html = html.replace(/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi, '');
|
||||
html = html.replace(/<iframe\b[^>]*>[\s\S]*?<\/iframe>/gi, '');
|
||||
html = html.replace(/<object\b[^>]*>[\s\S]*?<\/object>/gi, '');
|
||||
html = html.replace(/<embed\b[^>]*\/?>/gi, '');
|
||||
html = html.replace(/<form\b[^>]*>[\s\S]*?<\/form>/gi, '');
|
||||
html = html.replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, '');
|
||||
html = html.replace(/<link\b[^>]*\/?>/gi, '');
|
||||
html = html.replace(/<base\b[^>]*\/?>/gi, '');
|
||||
html = html.replace(/<meta\b[^>]*\/?>/gi, '');
|
||||
// Remove event handler attributes (onclick, onerror, onload, etc.)
|
||||
html = html.replace(/\s+on\w+\s*=\s*"[^"]*"/gi, '');
|
||||
html = html.replace(/\s+on\w+\s*=\s*'[^']*'/gi, '');
|
||||
html = html.replace(/\s+on\w+\s*=\s*[^\s>]+/gi, '');
|
||||
// Remove javascript: and data: URLs in href/src attributes
|
||||
html = html.replace(/(href|src|action)\s*=\s*["']?\s*javascript\s*:/gi, '$1="');
|
||||
html = html.replace(/(href|src|action)\s*=\s*["']?\s*data\s*:/gi, '$1="');
|
||||
return html;
|
||||
}
|
||||
|
||||
function copyCodeBlock(btn) {
|
||||
const pre = btn.parentElement;
|
||||
const code = pre.querySelector('code');
|
||||
|
||||
+13
-2
@@ -71,8 +71,17 @@ pub async fn handle_ws_connection(socket: WebSocket, state: Arc<GatewayState>) {
|
||||
}
|
||||
let tracker_for_drop = state.ws_tracker.clone();
|
||||
|
||||
// Subscribe to broadcast events (same source as SSE)
|
||||
let mut event_stream = Box::pin(state.sse.subscribe_raw());
|
||||
// Subscribe to broadcast events (same source as SSE).
|
||||
// Reject if we've hit the connection limit.
|
||||
let Some(raw_stream) = state.sse.subscribe_raw() else {
|
||||
tracing::warn!("WebSocket rejected: too many connections");
|
||||
// Decrement the WS tracker we already incremented above.
|
||||
if let Some(ref tracker) = tracker_for_drop {
|
||||
tracker.decrement();
|
||||
}
|
||||
return;
|
||||
};
|
||||
let mut event_stream = Box::pin(raw_stream);
|
||||
|
||||
// Channel for the sender task to receive messages from both
|
||||
// the broadcast stream and any direct sends (like Pong)
|
||||
@@ -476,6 +485,8 @@ mod tests {
|
||||
user_id: "test".to_string(),
|
||||
shutdown_tx: tokio::sync::RwLock::new(None),
|
||||
ws_tracker: Some(Arc::new(WsConnectionTracker::new())),
|
||||
llm_provider: None,
|
||||
chat_rate_limiter: crate::channels::web::server::RateLimiter::new(30, 60),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+22
-19
@@ -1,7 +1,9 @@
|
||||
//! Configuration management CLI commands.
|
||||
//!
|
||||
//! Commands for viewing and modifying settings.
|
||||
//! Settings are stored in PostgreSQL (env > DB > default).
|
||||
//! Settings are stored in the database (env > DB > default).
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use clap::Subcommand;
|
||||
|
||||
@@ -49,8 +51,8 @@ pub async fn run_config_command(cmd: ConfigCommand) -> anyhow::Result<()> {
|
||||
let _ = dotenvy::dotenv();
|
||||
|
||||
// Try to connect to the DB for settings access
|
||||
let store = match connect_store().await {
|
||||
Ok(s) => Some(s),
|
||||
let db: Option<Arc<dyn crate::db::Database>> = match connect_db().await {
|
||||
Ok(d) => Some(d),
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"Warning: Could not connect to database ({}), using disk fallback",
|
||||
@@ -60,29 +62,30 @@ pub async fn run_config_command(cmd: ConfigCommand) -> anyhow::Result<()> {
|
||||
}
|
||||
};
|
||||
|
||||
let db_ref = db.as_deref();
|
||||
match cmd {
|
||||
ConfigCommand::List { filter } => list_settings(store.as_ref(), filter).await,
|
||||
ConfigCommand::Get { path } => get_setting(store.as_ref(), &path).await,
|
||||
ConfigCommand::Set { path, value } => set_setting(store.as_ref(), &path, &value).await,
|
||||
ConfigCommand::Reset { path } => reset_setting(store.as_ref(), &path).await,
|
||||
ConfigCommand::Path => show_path(store.is_some()),
|
||||
ConfigCommand::List { filter } => list_settings(db_ref, filter).await,
|
||||
ConfigCommand::Get { path } => get_setting(db_ref, &path).await,
|
||||
ConfigCommand::Set { path, value } => set_setting(db_ref, &path, &value).await,
|
||||
ConfigCommand::Reset { path } => reset_setting(db_ref, &path).await,
|
||||
ConfigCommand::Path => show_path(db_ref.is_some()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Bootstrap a DB connection for config commands.
|
||||
async fn connect_store() -> anyhow::Result<crate::history::Store> {
|
||||
/// Bootstrap a DB connection for config commands (backend-agnostic).
|
||||
async fn connect_db() -> anyhow::Result<Arc<dyn crate::db::Database>> {
|
||||
let config = crate::config::Config::from_env()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||
let store = crate::history::Store::new(&config.database).await?;
|
||||
store.run_migrations().await?;
|
||||
Ok(store)
|
||||
crate::db::connect_from_config(&config.database)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))
|
||||
}
|
||||
|
||||
const DEFAULT_USER_ID: &str = "default";
|
||||
|
||||
/// Load settings: DB if available, else disk.
|
||||
async fn load_settings(store: Option<&crate::history::Store>) -> Settings {
|
||||
async fn load_settings(store: Option<&dyn crate::db::Database>) -> Settings {
|
||||
if let Some(store) = store {
|
||||
match store.get_all_settings(DEFAULT_USER_ID).await {
|
||||
Ok(map) if !map.is_empty() => return Settings::from_db_map(&map),
|
||||
@@ -94,7 +97,7 @@ async fn load_settings(store: Option<&crate::history::Store>) -> Settings {
|
||||
|
||||
/// List all settings.
|
||||
async fn list_settings(
|
||||
store: Option<&crate::history::Store>,
|
||||
store: Option<&dyn crate::db::Database>,
|
||||
filter: Option<String>,
|
||||
) -> anyhow::Result<()> {
|
||||
let settings = load_settings(store).await;
|
||||
@@ -126,7 +129,7 @@ async fn list_settings(
|
||||
}
|
||||
|
||||
/// Get a specific setting.
|
||||
async fn get_setting(store: Option<&crate::history::Store>, path: &str) -> anyhow::Result<()> {
|
||||
async fn get_setting(store: Option<&dyn crate::db::Database>, path: &str) -> anyhow::Result<()> {
|
||||
let settings = load_settings(store).await;
|
||||
|
||||
match settings.get(path) {
|
||||
@@ -142,7 +145,7 @@ async fn get_setting(store: Option<&crate::history::Store>, path: &str) -> anyho
|
||||
|
||||
/// Set a setting value.
|
||||
async fn set_setting(
|
||||
store: Option<&crate::history::Store>,
|
||||
store: Option<&dyn crate::db::Database>,
|
||||
path: &str,
|
||||
value: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
@@ -171,7 +174,7 @@ async fn set_setting(
|
||||
}
|
||||
|
||||
/// Reset a setting to default.
|
||||
async fn reset_setting(store: Option<&crate::history::Store>, path: &str) -> anyhow::Result<()> {
|
||||
async fn reset_setting(store: Option<&dyn crate::db::Database>, path: &str) -> anyhow::Result<()> {
|
||||
let default = Settings::default();
|
||||
let default_value = default
|
||||
.get(path)
|
||||
@@ -196,7 +199,7 @@ async fn reset_setting(store: Option<&crate::history::Store>, path: &str) -> any
|
||||
/// Show the settings storage info.
|
||||
fn show_path(has_db: bool) -> anyhow::Result<()> {
|
||||
if has_db {
|
||||
println!("Settings stored in: PostgreSQL (settings table)");
|
||||
println!("Settings stored in: database (settings table)");
|
||||
println!(
|
||||
"Bootstrap config: {}",
|
||||
crate::bootstrap::BootstrapConfig::default_path().display()
|
||||
|
||||
+82
-35
@@ -8,8 +8,10 @@ use std::sync::Arc;
|
||||
use clap::Subcommand;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::history::Store;
|
||||
use crate::secrets::{PostgresSecretsStore, SecretsCrypto, SecretsStore};
|
||||
use crate::db::Database;
|
||||
#[cfg(feature = "postgres")]
|
||||
use crate::secrets::PostgresSecretsStore;
|
||||
use crate::secrets::{SecretsCrypto, SecretsStore};
|
||||
use crate::tools::mcp::{
|
||||
McpClient, McpServerConfig, McpSessionManager, OAuthConfig,
|
||||
auth::{authorize_mcp_server, is_authenticated},
|
||||
@@ -172,10 +174,10 @@ async fn add_server(
|
||||
config.validate()?;
|
||||
|
||||
// Save (DB if available, else disk)
|
||||
let store = connect_store().await;
|
||||
let mut servers = load_servers(store.as_ref()).await?;
|
||||
let db = connect_db().await;
|
||||
let mut servers = load_servers(db.as_deref()).await?;
|
||||
servers.upsert(config);
|
||||
save_servers(store.as_ref(), &servers).await?;
|
||||
save_servers(db.as_deref(), &servers).await?;
|
||||
|
||||
println!();
|
||||
println!(" ✓ Added MCP server '{}'", name);
|
||||
@@ -193,12 +195,12 @@ async fn add_server(
|
||||
|
||||
/// Remove an MCP server.
|
||||
async fn remove_server(name: String) -> anyhow::Result<()> {
|
||||
let store = connect_store().await;
|
||||
let mut servers = load_servers(store.as_ref()).await?;
|
||||
let db = connect_db().await;
|
||||
let mut servers = load_servers(db.as_deref()).await?;
|
||||
if !servers.remove(&name) {
|
||||
anyhow::bail!("Server '{}' not found", name);
|
||||
}
|
||||
save_servers(store.as_ref(), &servers).await?;
|
||||
save_servers(db.as_deref(), &servers).await?;
|
||||
|
||||
println!();
|
||||
println!(" ✓ Removed MCP server '{}'", name);
|
||||
@@ -209,8 +211,8 @@ async fn remove_server(name: String) -> anyhow::Result<()> {
|
||||
|
||||
/// List configured MCP servers.
|
||||
async fn list_servers(verbose: bool) -> anyhow::Result<()> {
|
||||
let store = connect_store().await;
|
||||
let servers = load_servers(store.as_ref()).await?;
|
||||
let db = connect_db().await;
|
||||
let servers = load_servers(db.as_deref()).await?;
|
||||
|
||||
if servers.servers.is_empty() {
|
||||
println!();
|
||||
@@ -268,8 +270,8 @@ async fn list_servers(verbose: bool) -> anyhow::Result<()> {
|
||||
/// Authenticate with an MCP server.
|
||||
async fn auth_server(name: String, user_id: String) -> anyhow::Result<()> {
|
||||
// Get server config
|
||||
let store = connect_store().await;
|
||||
let servers = load_servers(store.as_ref()).await?;
|
||||
let db = connect_db().await;
|
||||
let servers = load_servers(db.as_deref()).await?;
|
||||
let server = servers
|
||||
.get(&name)
|
||||
.cloned()
|
||||
@@ -341,8 +343,8 @@ async fn auth_server(name: String, user_id: String) -> anyhow::Result<()> {
|
||||
/// Test connection to an MCP server.
|
||||
async fn test_server(name: String, user_id: String) -> anyhow::Result<()> {
|
||||
// Get server config
|
||||
let store = connect_store().await;
|
||||
let servers = load_servers(store.as_ref()).await?;
|
||||
let db = connect_db().await;
|
||||
let servers = load_servers(db.as_deref()).await?;
|
||||
let server = servers
|
||||
.get(&name)
|
||||
.cloned()
|
||||
@@ -437,8 +439,8 @@ async fn test_server(name: String, user_id: String) -> anyhow::Result<()> {
|
||||
|
||||
/// Toggle server enabled/disabled state.
|
||||
async fn toggle_server(name: String, enable: bool, disable: bool) -> anyhow::Result<()> {
|
||||
let store = connect_store().await;
|
||||
let mut servers = load_servers(store.as_ref()).await?;
|
||||
let db = connect_db().await;
|
||||
let mut servers = load_servers(db.as_deref()).await?;
|
||||
|
||||
let server = servers
|
||||
.get_mut(&name)
|
||||
@@ -453,7 +455,7 @@ async fn toggle_server(name: String, enable: bool, disable: bool) -> anyhow::Res
|
||||
};
|
||||
|
||||
server.enabled = new_state;
|
||||
save_servers(store.as_ref(), &servers).await?;
|
||||
save_servers(db.as_deref(), &servers).await?;
|
||||
|
||||
let status = if new_state { "enabled" } else { "disabled" };
|
||||
println!();
|
||||
@@ -465,18 +467,16 @@ async fn toggle_server(name: String, enable: bool, disable: bool) -> anyhow::Res
|
||||
|
||||
const DEFAULT_USER_ID: &str = "default";
|
||||
|
||||
/// Try to connect to the database store for DB-backed config.
|
||||
async fn connect_store() -> Option<Store> {
|
||||
/// Try to connect to the database (backend-agnostic).
|
||||
async fn connect_db() -> Option<Arc<dyn Database>> {
|
||||
let config = Config::from_env().await.ok()?;
|
||||
let store = Store::new(&config.database).await.ok()?;
|
||||
store.run_migrations().await.ok()?;
|
||||
Some(store)
|
||||
crate::db::connect_from_config(&config.database).await.ok()
|
||||
}
|
||||
|
||||
/// Load MCP servers (DB if available, else disk).
|
||||
async fn load_servers(store: Option<&Store>) -> Result<McpServersFile, config::ConfigError> {
|
||||
if let Some(store) = store {
|
||||
config::load_mcp_servers_from_db(store, DEFAULT_USER_ID).await
|
||||
async fn load_servers(db: Option<&dyn Database>) -> Result<McpServersFile, config::ConfigError> {
|
||||
if let Some(db) = db {
|
||||
config::load_mcp_servers_from_db(db, DEFAULT_USER_ID).await
|
||||
} else {
|
||||
config::load_mcp_servers().await
|
||||
}
|
||||
@@ -484,11 +484,11 @@ async fn load_servers(store: Option<&Store>) -> Result<McpServersFile, config::C
|
||||
|
||||
/// Save MCP servers (DB if available, else disk).
|
||||
async fn save_servers(
|
||||
store: Option<&Store>,
|
||||
db: Option<&dyn Database>,
|
||||
servers: &McpServersFile,
|
||||
) -> Result<(), config::ConfigError> {
|
||||
if let Some(store) = store {
|
||||
config::save_mcp_servers_to_db(store, DEFAULT_USER_ID, servers).await
|
||||
if let Some(db) = db {
|
||||
config::save_mcp_servers_to_db(db, DEFAULT_USER_ID, servers).await
|
||||
} else {
|
||||
config::save_mcp_servers(servers).await
|
||||
}
|
||||
@@ -504,14 +504,61 @@ async fn get_secrets_store() -> anyhow::Result<Arc<dyn SecretsStore + Send + Syn
|
||||
)
|
||||
})?;
|
||||
|
||||
let store = Store::new(&config.database).await?;
|
||||
store.run_migrations().await?;
|
||||
|
||||
let crypto = SecretsCrypto::new(master_key.clone())?;
|
||||
Ok(Arc::new(PostgresSecretsStore::new(
|
||||
store.pool(),
|
||||
Arc::new(crypto),
|
||||
)))
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
{
|
||||
let store = crate::history::Store::new(&config.database).await?;
|
||||
store.run_migrations().await?;
|
||||
Ok(Arc::new(PostgresSecretsStore::new(
|
||||
store.pool(),
|
||||
Arc::new(crypto),
|
||||
)))
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "libsql", not(feature = "postgres")))]
|
||||
{
|
||||
use crate::db::Database as _;
|
||||
use crate::db::libsql_backend::LibSqlBackend;
|
||||
use secrecy::ExposeSecret as _;
|
||||
|
||||
let default_path = crate::config::default_libsql_path();
|
||||
let db_path = config
|
||||
.database
|
||||
.libsql_path
|
||||
.as_deref()
|
||||
.unwrap_or(&default_path);
|
||||
|
||||
let backend = if let Some(ref url) = config.database.libsql_url {
|
||||
let token = config.database.libsql_auth_token.as_ref().ok_or_else(|| {
|
||||
anyhow::anyhow!("LIBSQL_AUTH_TOKEN is required when LIBSQL_URL is set")
|
||||
})?;
|
||||
LibSqlBackend::new_remote_replica(db_path, url, token.expose_secret())
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?
|
||||
} else {
|
||||
LibSqlBackend::new_local(db_path)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?
|
||||
};
|
||||
backend
|
||||
.run_migrations()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||
|
||||
return Ok(Arc::new(crate::secrets::LibSqlSecretsStore::new(
|
||||
backend.shared_db(),
|
||||
Arc::new(crypto),
|
||||
)));
|
||||
}
|
||||
|
||||
#[cfg(not(any(feature = "postgres", feature = "libsql")))]
|
||||
{
|
||||
let _ = crypto;
|
||||
anyhow::bail!(
|
||||
"No database backend available for secrets. Enable 'postgres' or 'libsql' feature."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
+26
-1
@@ -9,6 +9,30 @@ use clap::Subcommand;
|
||||
|
||||
use crate::workspace::{EmbeddingProvider, SearchConfig, Workspace};
|
||||
|
||||
/// Run a memory command using the Database trait (works with any backend).
|
||||
pub async fn run_memory_command_with_db(
|
||||
cmd: MemoryCommand,
|
||||
db: std::sync::Arc<dyn crate::db::Database>,
|
||||
embeddings: Option<Arc<dyn EmbeddingProvider>>,
|
||||
) -> anyhow::Result<()> {
|
||||
let mut workspace = Workspace::new_with_db("default", db);
|
||||
if let Some(emb) = embeddings {
|
||||
workspace = workspace.with_embeddings(emb);
|
||||
}
|
||||
|
||||
match cmd {
|
||||
MemoryCommand::Search { query, limit } => search(&workspace, &query, limit).await,
|
||||
MemoryCommand::Read { path } => read(&workspace, &path).await,
|
||||
MemoryCommand::Write {
|
||||
path,
|
||||
content,
|
||||
append,
|
||||
} => write(&workspace, &path, content, append).await,
|
||||
MemoryCommand::Tree { path, depth } => tree(&workspace, &path, depth).await,
|
||||
MemoryCommand::Status => status(&workspace).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug, Clone)]
|
||||
pub enum MemoryCommand {
|
||||
/// Search workspace memory (hybrid full-text + semantic)
|
||||
@@ -55,7 +79,8 @@ pub enum MemoryCommand {
|
||||
Status,
|
||||
}
|
||||
|
||||
/// Run a memory command.
|
||||
/// Run a memory command (PostgreSQL backend).
|
||||
#[cfg(feature = "postgres")]
|
||||
pub async fn run_memory_command(
|
||||
cmd: MemoryCommand,
|
||||
pool: deadpool_postgres::Pool,
|
||||
|
||||
+4
-1
@@ -18,7 +18,10 @@ mod tool;
|
||||
|
||||
pub use config::{ConfigCommand, run_config_command};
|
||||
pub use mcp::{McpCommand, run_mcp_command};
|
||||
pub use memory::{MemoryCommand, run_memory_command};
|
||||
pub use memory::MemoryCommand;
|
||||
#[cfg(feature = "postgres")]
|
||||
pub use memory::run_memory_command;
|
||||
pub use memory::run_memory_command_with_db;
|
||||
pub use pairing::{PairingCommand, run_pairing_command, run_pairing_command_with_store};
|
||||
pub use status::run_status_command;
|
||||
pub use tool::{ToolCommand, run_tool_command};
|
||||
|
||||
@@ -135,6 +135,7 @@ pub async fn run_status_command() -> anyhow::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
async fn check_database() -> anyhow::Result<()> {
|
||||
let _ = dotenvy::dotenv();
|
||||
let settings = Settings::load();
|
||||
@@ -167,6 +168,12 @@ async fn check_database() -> anyhow::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
async fn check_database() -> anyhow::Result<()> {
|
||||
// For non-postgres backends, just report configured
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn count_wasm_files(dir: &std::path::Path) -> usize {
|
||||
std::fs::read_dir(dir)
|
||||
.map(|entries| {
|
||||
|
||||
+63
-13
@@ -11,8 +11,11 @@ use clap::Subcommand;
|
||||
use tokio::fs;
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::history::Store;
|
||||
use crate::secrets::{CreateSecretParams, PostgresSecretsStore, SecretsCrypto, SecretsStore};
|
||||
#[allow(unused_imports)]
|
||||
use crate::db::Database;
|
||||
#[cfg(feature = "postgres")]
|
||||
use crate::secrets::PostgresSecretsStore;
|
||||
use crate::secrets::{CreateSecretParams, SecretsCrypto, SecretsStore};
|
||||
use crate::tools::wasm::{CapabilitiesFile, compute_binary_hash};
|
||||
|
||||
/// Default tools directory.
|
||||
@@ -722,11 +725,58 @@ async fn auth_tool(name: String, dir: Option<PathBuf>, user_id: String) -> anyho
|
||||
)
|
||||
})?;
|
||||
|
||||
let store = Store::new(&config.database).await?;
|
||||
store.run_migrations().await?;
|
||||
|
||||
let crypto = SecretsCrypto::new(master_key.clone())?;
|
||||
let secrets_store = Arc::new(PostgresSecretsStore::new(store.pool(), Arc::new(crypto)));
|
||||
|
||||
let secrets_store: Arc<dyn SecretsStore + Send + Sync> = {
|
||||
#[cfg(feature = "postgres")]
|
||||
{
|
||||
let store = crate::history::Store::new(&config.database).await?;
|
||||
store.run_migrations().await?;
|
||||
Arc::new(PostgresSecretsStore::new(store.pool(), Arc::new(crypto)))
|
||||
}
|
||||
#[cfg(all(feature = "libsql", not(feature = "postgres")))]
|
||||
{
|
||||
use crate::db::Database as _;
|
||||
use crate::db::libsql_backend::LibSqlBackend;
|
||||
use secrecy::ExposeSecret as _;
|
||||
|
||||
let default_path = crate::config::default_libsql_path();
|
||||
let db_path = config
|
||||
.database
|
||||
.libsql_path
|
||||
.as_deref()
|
||||
.unwrap_or(&default_path);
|
||||
|
||||
let backend = if let Some(ref url) = config.database.libsql_url {
|
||||
let token = config.database.libsql_auth_token.as_ref().ok_or_else(|| {
|
||||
anyhow::anyhow!("LIBSQL_AUTH_TOKEN is required when LIBSQL_URL is set")
|
||||
})?;
|
||||
LibSqlBackend::new_remote_replica(db_path, url, token.expose_secret())
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?
|
||||
} else {
|
||||
LibSqlBackend::new_local(db_path)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?
|
||||
};
|
||||
backend
|
||||
.run_migrations()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||
|
||||
Arc::new(crate::secrets::LibSqlSecretsStore::new(
|
||||
backend.shared_db(),
|
||||
Arc::new(crypto),
|
||||
))
|
||||
}
|
||||
#[cfg(not(any(feature = "postgres", feature = "libsql")))]
|
||||
{
|
||||
let _ = crypto;
|
||||
anyhow::bail!(
|
||||
"No database backend available for secrets. Enable 'postgres' or 'libsql' feature."
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// Check if already configured
|
||||
let already_configured = secrets_store
|
||||
@@ -772,13 +822,13 @@ async fn auth_tool(name: String, dir: Option<PathBuf>, user_id: String) -> anyho
|
||||
println!(" Validation failed: {}", e);
|
||||
println!();
|
||||
println!(" Falling back to manual entry...");
|
||||
return auth_tool_manual(&secrets_store, &user_id, &auth).await;
|
||||
return auth_tool_manual(secrets_store.as_ref(), &user_id, &auth).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save the token
|
||||
save_token(&secrets_store, &user_id, &auth, &token).await?;
|
||||
save_token(secrets_store.as_ref(), &user_id, &auth, &token).await?;
|
||||
print_success(display_name);
|
||||
return Ok(());
|
||||
}
|
||||
@@ -787,16 +837,16 @@ async fn auth_tool(name: String, dir: Option<PathBuf>, user_id: String) -> anyho
|
||||
|
||||
// Check for OAuth configuration
|
||||
if let Some(ref oauth) = auth.oauth {
|
||||
return auth_tool_oauth(&secrets_store, &user_id, &auth, oauth).await;
|
||||
return auth_tool_oauth(secrets_store.as_ref(), &user_id, &auth, oauth).await;
|
||||
}
|
||||
|
||||
// Fall back to manual entry
|
||||
auth_tool_manual(&secrets_store, &user_id, &auth).await
|
||||
auth_tool_manual(secrets_store.as_ref(), &user_id, &auth).await
|
||||
}
|
||||
|
||||
/// OAuth browser-based login flow.
|
||||
async fn auth_tool_oauth(
|
||||
store: &PostgresSecretsStore,
|
||||
store: &(dyn SecretsStore + Send + Sync),
|
||||
user_id: &str,
|
||||
auth: &crate::tools::wasm::AuthCapabilitySchema,
|
||||
oauth: &crate::tools::wasm::OAuthConfigSchema,
|
||||
@@ -1044,7 +1094,7 @@ async fn auth_tool_oauth(
|
||||
|
||||
/// Manual token entry flow.
|
||||
async fn auth_tool_manual(
|
||||
store: &PostgresSecretsStore,
|
||||
store: &(dyn SecretsStore + Send + Sync),
|
||||
user_id: &str,
|
||||
auth: &crate::tools::wasm::AuthCapabilitySchema,
|
||||
) -> anyhow::Result<()> {
|
||||
@@ -1217,7 +1267,7 @@ async fn validate_token(
|
||||
|
||||
/// Save token to secrets store.
|
||||
async fn save_token(
|
||||
store: &PostgresSecretsStore,
|
||||
store: &(dyn SecretsStore + Send + Sync),
|
||||
user_id: &str,
|
||||
auth: &crate::tools::wasm::AuthCapabilitySchema,
|
||||
token: &str,
|
||||
|
||||
+137
-1
@@ -38,7 +38,7 @@ impl Config {
|
||||
/// Priority: env var > DB settings > default.
|
||||
/// This is the primary way to load config after DB is connected.
|
||||
pub async fn from_db(
|
||||
store: &crate::history::Store,
|
||||
store: &dyn crate::db::Database,
|
||||
user_id: &str,
|
||||
bootstrap: &crate::bootstrap::BootstrapConfig,
|
||||
) -> Result<Self, ConfigError> {
|
||||
@@ -134,17 +134,72 @@ impl TunnelConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Which database backend to use.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum DatabaseBackend {
|
||||
/// PostgreSQL via deadpool-postgres (default).
|
||||
#[default]
|
||||
Postgres,
|
||||
/// libSQL/Turso embedded database.
|
||||
LibSql,
|
||||
}
|
||||
|
||||
impl std::str::FromStr for DatabaseBackend {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s.to_lowercase().as_str() {
|
||||
"postgres" | "postgresql" | "pg" => Ok(Self::Postgres),
|
||||
"libsql" | "turso" | "sqlite" => Ok(Self::LibSql),
|
||||
_ => Err(format!(
|
||||
"invalid database backend '{}', expected 'postgres' or 'libsql'",
|
||||
s
|
||||
)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Database configuration.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DatabaseConfig {
|
||||
/// Which backend to use (default: Postgres).
|
||||
pub backend: DatabaseBackend,
|
||||
|
||||
// -- PostgreSQL fields --
|
||||
pub url: SecretString,
|
||||
pub pool_size: usize,
|
||||
|
||||
// -- libSQL fields --
|
||||
/// Path to local libSQL database file (default: ~/.ironclaw/ironclaw.db).
|
||||
pub libsql_path: Option<PathBuf>,
|
||||
/// Turso cloud URL for remote sync (optional).
|
||||
pub libsql_url: Option<String>,
|
||||
/// Turso auth token (required when libsql_url is set).
|
||||
pub libsql_auth_token: Option<SecretString>,
|
||||
}
|
||||
|
||||
impl DatabaseConfig {
|
||||
fn resolve(bootstrap: &crate::bootstrap::BootstrapConfig) -> Result<Self, ConfigError> {
|
||||
let backend: DatabaseBackend = if let Some(b) = optional_env("DATABASE_BACKEND")? {
|
||||
b.parse().map_err(|e| ConfigError::InvalidValue {
|
||||
key: "DATABASE_BACKEND".to_string(),
|
||||
message: e,
|
||||
})?
|
||||
} else {
|
||||
DatabaseBackend::default()
|
||||
};
|
||||
|
||||
// PostgreSQL URL is required only when using the postgres backend.
|
||||
// For libsql backend, default to an empty placeholder.
|
||||
let url = optional_env("DATABASE_URL")?
|
||||
.or_else(|| bootstrap.database_url.clone())
|
||||
.or_else(|| {
|
||||
if backend == DatabaseBackend::LibSql {
|
||||
Some("unused://libsql".to_string())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.ok_or_else(|| ConfigError::MissingRequired {
|
||||
key: "database_url".to_string(),
|
||||
hint: "Run 'ironclaw onboard' or set DATABASE_URL environment variable".to_string(),
|
||||
@@ -160,9 +215,31 @@ impl DatabaseConfig {
|
||||
.or(bootstrap.database_pool_size)
|
||||
.unwrap_or(10);
|
||||
|
||||
let libsql_path = optional_env("LIBSQL_PATH")?.map(PathBuf::from).or_else(|| {
|
||||
if backend == DatabaseBackend::LibSql {
|
||||
Some(default_libsql_path())
|
||||
} else {
|
||||
None
|
||||
}
|
||||
});
|
||||
|
||||
let libsql_url = optional_env("LIBSQL_URL")?;
|
||||
let libsql_auth_token = optional_env("LIBSQL_AUTH_TOKEN")?.map(SecretString::from);
|
||||
|
||||
if libsql_url.is_some() && libsql_auth_token.is_none() {
|
||||
return Err(ConfigError::MissingRequired {
|
||||
key: "LIBSQL_AUTH_TOKEN".to_string(),
|
||||
hint: "LIBSQL_AUTH_TOKEN is required when LIBSQL_URL is set".to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
backend,
|
||||
url: SecretString::from(url),
|
||||
pool_size,
|
||||
libsql_path,
|
||||
libsql_url,
|
||||
libsql_auth_token,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -172,6 +249,14 @@ impl DatabaseConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Default libSQL database path (~/.ironclaw/ironclaw.db).
|
||||
pub fn default_libsql_path() -> PathBuf {
|
||||
dirs::home_dir()
|
||||
.unwrap_or_else(|| PathBuf::from("."))
|
||||
.join(".ironclaw")
|
||||
.join("ironclaw.db")
|
||||
}
|
||||
|
||||
/// Which LLM backend to use.
|
||||
///
|
||||
/// Defaults to `NearAi` to keep IronClaw close to the NEAR ecosystem.
|
||||
@@ -1174,6 +1259,36 @@ pub struct ClaudeCodeConfig {
|
||||
pub max_turns: u32,
|
||||
/// Memory limit in MB for Claude Code containers (heavier than workers).
|
||||
pub memory_limit_mb: u64,
|
||||
/// Allowed tool patterns for Claude Code permission settings.
|
||||
///
|
||||
/// Written to `/workspace/.claude/settings.json` before spawning the CLI.
|
||||
/// Provides defense-in-depth: only explicitly listed tools are auto-approved.
|
||||
/// Any new/unknown tools would require interactive approval (which times out
|
||||
/// in the non-interactive container, failing safely).
|
||||
///
|
||||
/// Patterns follow Claude Code syntax: `"Bash(*)"`, `"Read"`, `"Edit(*)"`, etc.
|
||||
pub allowed_tools: Vec<String>,
|
||||
}
|
||||
|
||||
/// Default allowed tools for Claude Code inside containers.
|
||||
///
|
||||
/// These cover all standard Claude Code tools needed for autonomous operation.
|
||||
/// The Docker container provides the primary security boundary; this allowlist
|
||||
/// provides defense-in-depth by preventing any future unknown tools from being
|
||||
/// silently auto-approved.
|
||||
fn default_claude_code_allowed_tools() -> Vec<String> {
|
||||
[
|
||||
"Bash(*)",
|
||||
"Read",
|
||||
"Edit(*)",
|
||||
"Glob",
|
||||
"Grep",
|
||||
"WebFetch(*)",
|
||||
"Task(*)",
|
||||
]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl Default for ClaudeCodeConfig {
|
||||
@@ -1186,11 +1301,24 @@ impl Default for ClaudeCodeConfig {
|
||||
model: "sonnet".to_string(),
|
||||
max_turns: 50,
|
||||
memory_limit_mb: 4096,
|
||||
allowed_tools: default_claude_code_allowed_tools(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ClaudeCodeConfig {
|
||||
/// Load from environment variables only (used inside containers where
|
||||
/// there is no database or full config).
|
||||
pub fn from_env() -> Self {
|
||||
match Self::resolve() {
|
||||
Ok(c) => c,
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to resolve ClaudeCodeConfig: {e}, using defaults");
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve() -> Result<Self, ConfigError> {
|
||||
let defaults = Self::default();
|
||||
Ok(Self {
|
||||
@@ -1211,6 +1339,14 @@ impl ClaudeCodeConfig {
|
||||
"CLAUDE_CODE_MEMORY_LIMIT_MB",
|
||||
defaults.memory_limit_mb,
|
||||
)?,
|
||||
allowed_tools: optional_env("CLAUDE_CODE_ALLOWED_TOOLS")?
|
||||
.map(|s| {
|
||||
s.split(',')
|
||||
.map(|t| t.trim().to_string())
|
||||
.filter(|t| !t.is_empty())
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or(defaults.allowed_tools),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,20 +45,21 @@ impl ContextManager {
|
||||
title: impl Into<String>,
|
||||
description: impl Into<String>,
|
||||
) -> Result<Uuid, JobError> {
|
||||
let contexts = self.contexts.read().await;
|
||||
// Hold write lock for the entire check-insert to prevent TOCTOU races
|
||||
// where two concurrent calls both pass the active_count check.
|
||||
let mut contexts = self.contexts.write().await;
|
||||
let active_count = contexts.values().filter(|c| c.state.is_active()).count();
|
||||
|
||||
if active_count >= self.max_jobs {
|
||||
return Err(JobError::MaxJobsExceeded { max: self.max_jobs });
|
||||
}
|
||||
drop(contexts);
|
||||
|
||||
let context = JobContext::with_user(user_id, title, description);
|
||||
let job_id = context.job_id;
|
||||
contexts.insert(job_id, context);
|
||||
drop(contexts);
|
||||
|
||||
let memory = Memory::new(job_id);
|
||||
|
||||
self.contexts.write().await.insert(job_id, context);
|
||||
self.memories.write().await.insert(job_id, memory);
|
||||
|
||||
Ok(job_id)
|
||||
|
||||
@@ -119,6 +119,10 @@ pub struct JobContext {
|
||||
pub estimated_duration: Option<Duration>,
|
||||
/// Actual cost so far.
|
||||
pub actual_cost: Decimal,
|
||||
/// Total tokens consumed by LLM calls in this job.
|
||||
pub total_tokens_used: u64,
|
||||
/// Maximum tokens allowed per job (0 = unlimited).
|
||||
pub max_tokens: u64,
|
||||
/// When the job was created.
|
||||
pub created_at: DateTime<Utc>,
|
||||
/// When the job was started.
|
||||
@@ -159,6 +163,8 @@ impl JobContext {
|
||||
estimated_cost: None,
|
||||
estimated_duration: None,
|
||||
actual_cost: Decimal::ZERO,
|
||||
total_tokens_used: 0,
|
||||
max_tokens: 0,
|
||||
created_at: Utc::now(),
|
||||
started_at: None,
|
||||
completed_at: None,
|
||||
@@ -189,6 +195,14 @@ impl JobContext {
|
||||
};
|
||||
|
||||
self.transitions.push(transition);
|
||||
|
||||
// Cap transition history to prevent unbounded memory growth
|
||||
const MAX_TRANSITIONS: usize = 200;
|
||||
if self.transitions.len() > MAX_TRANSITIONS {
|
||||
let drain_count = self.transitions.len() - MAX_TRANSITIONS;
|
||||
self.transitions.drain(..drain_count);
|
||||
}
|
||||
|
||||
self.state = new_state;
|
||||
|
||||
// Update timestamps
|
||||
@@ -210,6 +224,29 @@ impl JobContext {
|
||||
self.actual_cost += cost;
|
||||
}
|
||||
|
||||
/// Record token usage from an LLM call. Returns an error string if the
|
||||
/// token budget has been exceeded after this addition.
|
||||
pub fn add_tokens(&mut self, tokens: u64) -> Result<(), String> {
|
||||
self.total_tokens_used += tokens;
|
||||
if self.max_tokens > 0 && self.total_tokens_used > self.max_tokens {
|
||||
Err(format!(
|
||||
"Token budget exceeded: used {} of {} allowed tokens",
|
||||
self.total_tokens_used, self.max_tokens
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Check whether the monetary budget has been exceeded.
|
||||
pub fn budget_exceeded(&self) -> bool {
|
||||
if let Some(ref budget) = self.budget {
|
||||
self.actual_cost > *budget
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the duration since the job started.
|
||||
pub fn elapsed(&self) -> Option<Duration> {
|
||||
self.started_at.map(|start| {
|
||||
@@ -274,6 +311,57 @@ mod tests {
|
||||
assert_eq!(ctx.state, JobState::Completed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_transition_history_capped() {
|
||||
let mut ctx = JobContext::new("Test", "Transition cap test");
|
||||
// Cycle through Pending -> InProgress -> Stuck -> InProgress -> Stuck ...
|
||||
ctx.transition_to(JobState::InProgress, None).unwrap();
|
||||
for i in 0..250 {
|
||||
ctx.mark_stuck(format!("stuck {}", i)).unwrap();
|
||||
ctx.attempt_recovery().unwrap();
|
||||
}
|
||||
// 1 initial + 250*2 = 501 transitions, should be capped at 200
|
||||
assert!(
|
||||
ctx.transitions.len() <= 200,
|
||||
"transitions should be capped at 200, got {}",
|
||||
ctx.transitions.len()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_tokens_enforces_budget() {
|
||||
let mut ctx = JobContext::new("Test", "Budget test");
|
||||
ctx.max_tokens = 1000;
|
||||
assert!(ctx.add_tokens(500).is_ok());
|
||||
assert_eq!(ctx.total_tokens_used, 500);
|
||||
assert!(ctx.add_tokens(600).is_err());
|
||||
assert_eq!(ctx.total_tokens_used, 1100); // tokens still recorded
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_add_tokens_unlimited() {
|
||||
let mut ctx = JobContext::new("Test", "No budget");
|
||||
// max_tokens = 0 means unlimited
|
||||
assert!(ctx.add_tokens(1_000_000).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_budget_exceeded() {
|
||||
let mut ctx = JobContext::new("Test", "Money test");
|
||||
ctx.budget = Some(Decimal::new(100, 0)); // $100
|
||||
assert!(!ctx.budget_exceeded());
|
||||
ctx.add_cost(Decimal::new(50, 0));
|
||||
assert!(!ctx.budget_exceeded());
|
||||
ctx.add_cost(Decimal::new(60, 0));
|
||||
assert!(ctx.budget_exceeded());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_budget_exceeded_none() {
|
||||
let ctx = JobContext::new("Test", "No budget");
|
||||
assert!(!ctx.budget_exceeded()); // No budget = never exceeded
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_stuck_recovery() {
|
||||
let mut ctx = JobContext::new("Test", "Test job");
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,549 @@
|
||||
//! SQLite-dialect migrations for the libSQL/Turso backend.
|
||||
//!
|
||||
//! Consolidates all PostgreSQL migrations (V1-V8) into a single SQLite-compatible
|
||||
//! schema. Run once on database creation; idempotent via `IF NOT EXISTS`.
|
||||
|
||||
/// Consolidated schema for libSQL.
|
||||
///
|
||||
/// Translates PostgreSQL types and features:
|
||||
/// - `UUID` -> `TEXT` (store as hex string)
|
||||
/// - `TIMESTAMPTZ` -> `TEXT` (ISO-8601)
|
||||
/// - `JSONB` -> `TEXT` (JSON encoded)
|
||||
/// - `BYTEA` -> `BLOB`
|
||||
/// - `NUMERIC` -> `TEXT` (preserve precision for rust_decimal)
|
||||
/// - `TEXT[]` -> `TEXT` (JSON array)
|
||||
/// - `VECTOR(1536)` -> `F32_BLOB(1536)` (libsql native)
|
||||
/// - `TSVECTOR` -> FTS5 virtual table
|
||||
/// - `BIGSERIAL` -> `INTEGER PRIMARY KEY AUTOINCREMENT`
|
||||
/// - PL/pgSQL functions -> SQLite triggers
|
||||
pub const SCHEMA: &str = r#"
|
||||
|
||||
-- ==================== Migration tracking ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS _migrations (
|
||||
version INTEGER PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- ==================== Conversations ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS conversations (
|
||||
id TEXT PRIMARY KEY,
|
||||
channel TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
thread_id TEXT,
|
||||
started_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
last_activity TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
metadata TEXT NOT NULL DEFAULT '{}'
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_conversations_channel ON conversations(channel);
|
||||
CREATE INDEX IF NOT EXISTS idx_conversations_user ON conversations(user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_conversations_last_activity ON conversations(last_activity);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS conversation_messages (
|
||||
id TEXT PRIMARY KEY,
|
||||
conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
|
||||
role TEXT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_conversation_messages_conversation
|
||||
ON conversation_messages(conversation_id);
|
||||
|
||||
-- ==================== Agent Jobs ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS agent_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
marketplace_job_id TEXT,
|
||||
conversation_id TEXT REFERENCES conversations(id),
|
||||
title TEXT NOT NULL,
|
||||
description TEXT NOT NULL,
|
||||
category TEXT,
|
||||
status TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
user_id TEXT NOT NULL DEFAULT 'default',
|
||||
project_dir TEXT,
|
||||
job_mode TEXT NOT NULL DEFAULT 'worker',
|
||||
budget_amount TEXT,
|
||||
budget_token TEXT,
|
||||
bid_amount TEXT,
|
||||
estimated_cost TEXT,
|
||||
estimated_time_secs INTEGER,
|
||||
estimated_value TEXT,
|
||||
actual_cost TEXT,
|
||||
actual_time_secs INTEGER,
|
||||
success INTEGER,
|
||||
failure_reason TEXT,
|
||||
stuck_since TEXT,
|
||||
repair_attempts INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
started_at TEXT,
|
||||
completed_at TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_jobs_status ON agent_jobs(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_jobs_marketplace ON agent_jobs(marketplace_job_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_jobs_conversation ON agent_jobs(conversation_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_jobs_source ON agent_jobs(source);
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_jobs_user ON agent_jobs(user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_jobs_created ON agent_jobs(created_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS job_actions (
|
||||
id TEXT PRIMARY KEY,
|
||||
job_id TEXT NOT NULL REFERENCES agent_jobs(id) ON DELETE CASCADE,
|
||||
sequence_num INTEGER NOT NULL,
|
||||
tool_name TEXT NOT NULL,
|
||||
input TEXT NOT NULL,
|
||||
output_raw TEXT,
|
||||
output_sanitized TEXT,
|
||||
sanitization_warnings TEXT,
|
||||
cost TEXT,
|
||||
duration_ms INTEGER,
|
||||
success INTEGER NOT NULL,
|
||||
error_message TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE(job_id, sequence_num)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_job_actions_job_id ON job_actions(job_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_job_actions_tool ON job_actions(tool_name);
|
||||
|
||||
-- ==================== Dynamic Tools ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS dynamic_tools (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
description TEXT NOT NULL,
|
||||
parameters_schema TEXT NOT NULL,
|
||||
code TEXT NOT NULL,
|
||||
sandbox_config TEXT NOT NULL,
|
||||
created_by_job_id TEXT REFERENCES agent_jobs(id),
|
||||
success_count INTEGER NOT NULL DEFAULT 0,
|
||||
failure_count INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_dynamic_tools_status ON dynamic_tools(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_dynamic_tools_name ON dynamic_tools(name);
|
||||
|
||||
-- ==================== LLM Calls ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS llm_calls (
|
||||
id TEXT PRIMARY KEY,
|
||||
job_id TEXT REFERENCES agent_jobs(id) ON DELETE CASCADE,
|
||||
conversation_id TEXT REFERENCES conversations(id),
|
||||
provider TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
input_tokens INTEGER NOT NULL,
|
||||
output_tokens INTEGER NOT NULL,
|
||||
cost TEXT NOT NULL,
|
||||
purpose TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_llm_calls_job ON llm_calls(job_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_llm_calls_conversation ON llm_calls(conversation_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_llm_calls_provider ON llm_calls(provider);
|
||||
|
||||
-- ==================== Estimation ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS estimation_snapshots (
|
||||
id TEXT PRIMARY KEY,
|
||||
job_id TEXT NOT NULL REFERENCES agent_jobs(id) ON DELETE CASCADE,
|
||||
category TEXT NOT NULL,
|
||||
tool_names TEXT NOT NULL DEFAULT '[]',
|
||||
estimated_cost TEXT NOT NULL,
|
||||
actual_cost TEXT,
|
||||
estimated_time_secs INTEGER NOT NULL,
|
||||
actual_time_secs INTEGER,
|
||||
estimated_value TEXT NOT NULL,
|
||||
actual_value TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_estimation_category ON estimation_snapshots(category);
|
||||
CREATE INDEX IF NOT EXISTS idx_estimation_job ON estimation_snapshots(job_id);
|
||||
|
||||
-- ==================== Self Repair ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS repair_attempts (
|
||||
id TEXT PRIMARY KEY,
|
||||
target_type TEXT NOT NULL,
|
||||
target_id TEXT NOT NULL,
|
||||
diagnosis TEXT NOT NULL,
|
||||
action_taken TEXT NOT NULL,
|
||||
success INTEGER NOT NULL,
|
||||
error_message TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_repair_attempts_target ON repair_attempts(target_type, target_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_repair_attempts_created ON repair_attempts(created_at);
|
||||
|
||||
-- ==================== Workspace: Memory Documents ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memory_documents (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
agent_id TEXT,
|
||||
path TEXT NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
metadata TEXT NOT NULL DEFAULT '{}',
|
||||
UNIQUE (user_id, agent_id, path)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_documents_user ON memory_documents(user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_documents_path ON memory_documents(user_id, path);
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_documents_updated ON memory_documents(updated_at DESC);
|
||||
|
||||
-- Trigger to auto-update updated_at on memory_documents
|
||||
CREATE TRIGGER IF NOT EXISTS update_memory_documents_updated_at
|
||||
AFTER UPDATE ON memory_documents
|
||||
FOR EACH ROW
|
||||
WHEN NEW.updated_at = OLD.updated_at
|
||||
BEGIN
|
||||
UPDATE memory_documents SET updated_at = datetime('now') WHERE id = NEW.id;
|
||||
END;
|
||||
|
||||
-- ==================== Workspace: Memory Chunks ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS memory_chunks (
|
||||
_rowid INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
id TEXT NOT NULL UNIQUE,
|
||||
document_id TEXT NOT NULL REFERENCES memory_documents(id) ON DELETE CASCADE,
|
||||
chunk_index INTEGER NOT NULL,
|
||||
content TEXT NOT NULL,
|
||||
embedding F32_BLOB(1536),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE (document_id, chunk_index)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_chunks_document ON memory_chunks(document_id);
|
||||
|
||||
-- Vector index for semantic search (libSQL native)
|
||||
CREATE INDEX IF NOT EXISTS idx_memory_chunks_embedding
|
||||
ON memory_chunks (libsql_vector_idx(embedding));
|
||||
|
||||
-- FTS5 virtual table for full-text search
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS memory_chunks_fts USING fts5(
|
||||
content,
|
||||
content='memory_chunks',
|
||||
content_rowid='_rowid'
|
||||
);
|
||||
|
||||
-- Triggers to keep FTS5 in sync with memory_chunks
|
||||
CREATE TRIGGER IF NOT EXISTS memory_chunks_fts_insert AFTER INSERT ON memory_chunks BEGIN
|
||||
INSERT INTO memory_chunks_fts(rowid, content) VALUES (new._rowid, new.content);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS memory_chunks_fts_delete AFTER DELETE ON memory_chunks BEGIN
|
||||
INSERT INTO memory_chunks_fts(memory_chunks_fts, rowid, content)
|
||||
VALUES ('delete', old._rowid, old.content);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS memory_chunks_fts_update AFTER UPDATE ON memory_chunks BEGIN
|
||||
INSERT INTO memory_chunks_fts(memory_chunks_fts, rowid, content)
|
||||
VALUES ('delete', old._rowid, old.content);
|
||||
INSERT INTO memory_chunks_fts(rowid, content) VALUES (new._rowid, new.content);
|
||||
END;
|
||||
|
||||
-- ==================== Workspace: Heartbeat State ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS heartbeat_state (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
agent_id TEXT,
|
||||
last_run TEXT,
|
||||
next_run TEXT,
|
||||
interval_seconds INTEGER NOT NULL DEFAULT 1800,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
||||
last_checks TEXT NOT NULL DEFAULT '{}',
|
||||
UNIQUE (user_id, agent_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_heartbeat_user ON heartbeat_state(user_id);
|
||||
|
||||
-- ==================== Secrets ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS secrets (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
encrypted_value BLOB NOT NULL,
|
||||
key_salt BLOB NOT NULL,
|
||||
provider TEXT,
|
||||
expires_at TEXT,
|
||||
last_used_at TEXT,
|
||||
usage_count INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE (user_id, name)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_secrets_user ON secrets(user_id);
|
||||
|
||||
-- ==================== WASM Tools ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS wasm_tools (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
name TEXT NOT NULL,
|
||||
version TEXT NOT NULL DEFAULT '1.0.0',
|
||||
description TEXT NOT NULL,
|
||||
wasm_binary BLOB NOT NULL,
|
||||
binary_hash BLOB NOT NULL,
|
||||
parameters_schema TEXT NOT NULL,
|
||||
source_url TEXT,
|
||||
trust_level TEXT NOT NULL DEFAULT 'user',
|
||||
status TEXT NOT NULL DEFAULT 'active',
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE (user_id, name, version)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_wasm_tools_user ON wasm_tools(user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_wasm_tools_name ON wasm_tools(user_id, name);
|
||||
CREATE INDEX IF NOT EXISTS idx_wasm_tools_status ON wasm_tools(status);
|
||||
|
||||
-- ==================== Tool Capabilities ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tool_capabilities (
|
||||
id TEXT PRIMARY KEY,
|
||||
wasm_tool_id TEXT NOT NULL REFERENCES wasm_tools(id) ON DELETE CASCADE,
|
||||
http_allowlist TEXT NOT NULL DEFAULT '[]',
|
||||
allowed_secrets TEXT NOT NULL DEFAULT '[]',
|
||||
tool_aliases TEXT NOT NULL DEFAULT '{}',
|
||||
requests_per_minute INTEGER NOT NULL DEFAULT 60,
|
||||
requests_per_hour INTEGER NOT NULL DEFAULT 1000,
|
||||
max_request_body_bytes INTEGER NOT NULL DEFAULT 1048576,
|
||||
max_response_body_bytes INTEGER NOT NULL DEFAULT 10485760,
|
||||
workspace_read_prefixes TEXT NOT NULL DEFAULT '[]',
|
||||
http_timeout_secs INTEGER NOT NULL DEFAULT 30,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE (wasm_tool_id)
|
||||
);
|
||||
|
||||
-- ==================== Leak Detection Patterns ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS leak_detection_patterns (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL UNIQUE,
|
||||
pattern TEXT NOT NULL,
|
||||
severity TEXT NOT NULL DEFAULT 'high',
|
||||
action TEXT NOT NULL DEFAULT 'block',
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- ==================== Rate Limit State ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tool_rate_limit_state (
|
||||
id TEXT PRIMARY KEY,
|
||||
wasm_tool_id TEXT NOT NULL REFERENCES wasm_tools(id) ON DELETE CASCADE,
|
||||
user_id TEXT NOT NULL,
|
||||
minute_window_start TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
minute_count INTEGER NOT NULL DEFAULT 0,
|
||||
hour_window_start TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
hour_count INTEGER NOT NULL DEFAULT 0,
|
||||
UNIQUE (wasm_tool_id, user_id)
|
||||
);
|
||||
|
||||
-- ==================== Secret Usage Audit Log ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS secret_usage_log (
|
||||
id TEXT PRIMARY KEY,
|
||||
secret_id TEXT NOT NULL REFERENCES secrets(id) ON DELETE CASCADE,
|
||||
wasm_tool_id TEXT REFERENCES wasm_tools(id) ON DELETE SET NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
target_host TEXT NOT NULL,
|
||||
target_path TEXT,
|
||||
success INTEGER NOT NULL,
|
||||
error_message TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_secret_usage_user ON secret_usage_log(user_id);
|
||||
|
||||
-- ==================== Leak Detection Events ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS leak_detection_events (
|
||||
id TEXT PRIMARY KEY,
|
||||
pattern_id TEXT REFERENCES leak_detection_patterns(id) ON DELETE SET NULL,
|
||||
wasm_tool_id TEXT REFERENCES wasm_tools(id) ON DELETE SET NULL,
|
||||
user_id TEXT NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
action_taken TEXT NOT NULL,
|
||||
context_preview TEXT,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
-- ==================== Tool Failures ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS tool_failures (
|
||||
id TEXT PRIMARY KEY,
|
||||
tool_name TEXT NOT NULL UNIQUE,
|
||||
error_message TEXT,
|
||||
error_count INTEGER DEFAULT 1,
|
||||
first_failure TEXT DEFAULT (datetime('now')),
|
||||
last_failure TEXT DEFAULT (datetime('now')),
|
||||
last_build_result TEXT,
|
||||
repaired_at TEXT,
|
||||
repair_attempts INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_tool_failures_name ON tool_failures(tool_name);
|
||||
|
||||
-- ==================== Job Events ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS job_events (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
job_id TEXT NOT NULL REFERENCES agent_jobs(id),
|
||||
event_type TEXT NOT NULL,
|
||||
data TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_job_events_job ON job_events(job_id, id);
|
||||
|
||||
-- ==================== Routines ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS routines (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
description TEXT NOT NULL DEFAULT '',
|
||||
user_id TEXT NOT NULL,
|
||||
enabled INTEGER NOT NULL DEFAULT 1,
|
||||
trigger_type TEXT NOT NULL,
|
||||
trigger_config TEXT NOT NULL,
|
||||
action_type TEXT NOT NULL,
|
||||
action_config TEXT NOT NULL,
|
||||
cooldown_secs INTEGER NOT NULL DEFAULT 300,
|
||||
max_concurrent INTEGER NOT NULL DEFAULT 1,
|
||||
dedup_window_secs INTEGER,
|
||||
notify_channel TEXT,
|
||||
notify_user TEXT NOT NULL DEFAULT 'default',
|
||||
notify_on_success INTEGER NOT NULL DEFAULT 0,
|
||||
notify_on_failure INTEGER NOT NULL DEFAULT 1,
|
||||
notify_on_attention INTEGER NOT NULL DEFAULT 1,
|
||||
state TEXT NOT NULL DEFAULT '{}',
|
||||
last_run_at TEXT,
|
||||
next_fire_at TEXT,
|
||||
run_count INTEGER NOT NULL DEFAULT 0,
|
||||
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
UNIQUE (user_id, name)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_routines_user ON routines(user_id);
|
||||
|
||||
-- ==================== Routine Runs ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS routine_runs (
|
||||
id TEXT PRIMARY KEY,
|
||||
routine_id TEXT NOT NULL REFERENCES routines(id) ON DELETE CASCADE,
|
||||
trigger_type TEXT NOT NULL,
|
||||
trigger_detail TEXT,
|
||||
started_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
completed_at TEXT,
|
||||
status TEXT NOT NULL DEFAULT 'running',
|
||||
result_summary TEXT,
|
||||
tokens_used INTEGER,
|
||||
job_id TEXT REFERENCES agent_jobs(id),
|
||||
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_routine_runs_routine ON routine_runs(routine_id);
|
||||
|
||||
-- ==================== Settings ====================
|
||||
|
||||
CREATE TABLE IF NOT EXISTS settings (
|
||||
user_id TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
value TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL DEFAULT (datetime('now')),
|
||||
PRIMARY KEY (user_id, key)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_settings_user ON settings(user_id);
|
||||
|
||||
-- ==================== Missing indexes (parity with PostgreSQL) ====================
|
||||
|
||||
-- agent_jobs
|
||||
CREATE INDEX IF NOT EXISTS idx_agent_jobs_stuck ON agent_jobs(stuck_since);
|
||||
|
||||
-- secrets
|
||||
CREATE INDEX IF NOT EXISTS idx_secrets_provider ON secrets(provider);
|
||||
CREATE INDEX IF NOT EXISTS idx_secrets_expires ON secrets(expires_at);
|
||||
|
||||
-- wasm_tools
|
||||
CREATE INDEX IF NOT EXISTS idx_wasm_tools_trust ON wasm_tools(trust_level);
|
||||
|
||||
-- tool_capabilities
|
||||
CREATE INDEX IF NOT EXISTS idx_tool_capabilities_tool ON tool_capabilities(wasm_tool_id);
|
||||
|
||||
-- leak_detection_patterns
|
||||
CREATE INDEX IF NOT EXISTS idx_leak_patterns_enabled ON leak_detection_patterns(enabled);
|
||||
|
||||
-- tool_rate_limit_state
|
||||
CREATE INDEX IF NOT EXISTS idx_rate_limit_tool ON tool_rate_limit_state(wasm_tool_id);
|
||||
|
||||
-- secret_usage_log
|
||||
CREATE INDEX IF NOT EXISTS idx_secret_usage_secret ON secret_usage_log(secret_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_secret_usage_tool ON secret_usage_log(wasm_tool_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_secret_usage_created ON secret_usage_log(created_at DESC);
|
||||
|
||||
-- leak_detection_events
|
||||
CREATE INDEX IF NOT EXISTS idx_leak_events_pattern ON leak_detection_events(pattern_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_leak_events_tool ON leak_detection_events(wasm_tool_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_leak_events_user ON leak_detection_events(user_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_leak_events_created ON leak_detection_events(created_at DESC);
|
||||
|
||||
-- tool_failures
|
||||
CREATE INDEX IF NOT EXISTS idx_tool_failures_count ON tool_failures(error_count DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_tool_failures_unrepaired ON tool_failures(tool_name);
|
||||
|
||||
-- routines
|
||||
CREATE INDEX IF NOT EXISTS idx_routines_next_fire ON routines(next_fire_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_routines_event_triggers ON routines(user_id);
|
||||
|
||||
-- routine_runs
|
||||
CREATE INDEX IF NOT EXISTS idx_routine_runs_status ON routine_runs(status);
|
||||
|
||||
-- heartbeat_state
|
||||
CREATE INDEX IF NOT EXISTS idx_heartbeat_next_run ON heartbeat_state(next_run);
|
||||
|
||||
-- ==================== Seed data ====================
|
||||
|
||||
-- Pre-populate leak detection patterns (matches PostgreSQL V2 migration).
|
||||
INSERT OR IGNORE INTO leak_detection_patterns (id, name, pattern, severity, action, enabled, created_at) VALUES
|
||||
('550e8400-e29b-41d4-a716-446655440001', 'openai_api_key', 'sk-(?:proj-)?[a-zA-Z0-9]{20,}(?:T3BlbkFJ[a-zA-Z0-9_-]*)?', 'critical', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-446655440002', 'anthropic_api_key', 'sk-ant-api[a-zA-Z0-9_-]{90,}', 'critical', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-446655440003', 'aws_access_key', 'AKIA[0-9A-Z]{16}', 'critical', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-446655440004', 'aws_secret_key', '(?<![A-Za-z0-9/+=])[A-Za-z0-9/+=]{40}(?![A-Za-z0-9/+=])', 'high', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-446655440005', 'github_token', 'gh[pousr]_[A-Za-z0-9_]{36,}', 'critical', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-446655440006', 'github_fine_grained_pat', 'github_pat_[a-zA-Z0-9]{22}_[a-zA-Z0-9]{59}', 'critical', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-446655440007', 'stripe_api_key', 'sk_(?:live|test)_[a-zA-Z0-9]{24,}', 'critical', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-446655440008', 'nearai_session', 'sess_[a-zA-Z0-9]{32,}', 'critical', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-446655440009', 'bearer_token', 'Bearer\s+[a-zA-Z0-9_-]{20,}', 'high', 'redact', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-44665544000a', 'pem_private_key', '-----BEGIN\s+(?:RSA\s+)?PRIVATE\s+KEY-----', 'critical', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-44665544000b', 'ssh_private_key', '-----BEGIN\s+(?:OPENSSH|EC|DSA)\s+PRIVATE\s+KEY-----', 'critical', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-44665544000c', 'google_api_key', 'AIza[0-9A-Za-z_-]{35}', 'high', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-44665544000d', 'slack_token', 'xox[baprs]-[0-9a-zA-Z-]{10,}', 'high', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-44665544000e', 'discord_token', '[MN][A-Za-z\d]{23,}\.[\w-]{6}\.[\w-]{27}', 'high', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-44665544000f', 'twilio_api_key', 'SK[a-fA-F0-9]{32}', 'high', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-446655440010', 'sendgrid_api_key', 'SG\.[a-zA-Z0-9_-]{22}\.[a-zA-Z0-9_-]{43}', 'high', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-446655440011', 'mailchimp_api_key', '[a-f0-9]{32}-us[0-9]{1,2}', 'medium', 'block', 1, datetime('now')),
|
||||
('550e8400-e29b-41d4-a716-446655440012', 'high_entropy_hex', '(?<![a-fA-F0-9])[a-fA-F0-9]{64}(?![a-fA-F0-9])', 'medium', 'warn', 1, datetime('now'));
|
||||
|
||||
"#;
|
||||
+538
@@ -0,0 +1,538 @@
|
||||
//! Database abstraction layer.
|
||||
//!
|
||||
//! Provides a backend-agnostic `Database` trait that unifies all persistence
|
||||
//! operations. Two implementations exist behind feature flags:
|
||||
//!
|
||||
//! - `postgres` (default): Uses `deadpool-postgres` + `tokio-postgres`
|
||||
//! - `libsql`: Uses libSQL (Turso's SQLite fork) for embedded/edge deployment
|
||||
//!
|
||||
//! The existing `Store`, `Repository`, `SecretsStore`, and `WasmToolStore`
|
||||
//! types become thin wrappers that delegate to `Arc<dyn Database>`.
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
pub mod postgres;
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
pub mod libsql_backend;
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
pub mod libsql_migrations;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use rust_decimal::Decimal;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::agent::BrokenTool;
|
||||
use crate::agent::routine::{Routine, RoutineRun, RunStatus};
|
||||
use crate::context::{ActionRecord, JobContext, JobState};
|
||||
use crate::error::DatabaseError;
|
||||
use crate::error::WorkspaceError;
|
||||
use crate::history::{
|
||||
ConversationMessage, ConversationSummary, JobEventRecord, LlmCallRecord, SandboxJobRecord,
|
||||
SandboxJobSummary, SettingRow,
|
||||
};
|
||||
use crate::workspace::{MemoryChunk, MemoryDocument, WorkspaceEntry};
|
||||
use crate::workspace::{SearchConfig, SearchResult};
|
||||
|
||||
/// Create a database backend from configuration, run migrations, and return it.
|
||||
///
|
||||
/// This is the shared helper for CLI commands and other call sites that need
|
||||
/// a simple `Arc<dyn Database>` without retaining backend-specific handles
|
||||
/// (e.g., `pg_pool` or `libsql_conn` for the secrets store). The main agent
|
||||
/// startup in `main.rs` uses its own initialization block because it also
|
||||
/// captures those backend-specific handles.
|
||||
pub async fn connect_from_config(
|
||||
config: &crate::config::DatabaseConfig,
|
||||
) -> Result<Arc<dyn Database>, DatabaseError> {
|
||||
match config.backend {
|
||||
#[cfg(feature = "libsql")]
|
||||
crate::config::DatabaseBackend::LibSql => {
|
||||
use secrecy::ExposeSecret as _;
|
||||
|
||||
let default_path = crate::config::default_libsql_path();
|
||||
let db_path = config.libsql_path.as_deref().unwrap_or(&default_path);
|
||||
|
||||
let backend = if let Some(ref url) = config.libsql_url {
|
||||
let token = config.libsql_auth_token.as_ref().ok_or_else(|| {
|
||||
DatabaseError::Pool(
|
||||
"LIBSQL_AUTH_TOKEN required when LIBSQL_URL is set".to_string(),
|
||||
)
|
||||
})?;
|
||||
libsql_backend::LibSqlBackend::new_remote_replica(
|
||||
db_path,
|
||||
url,
|
||||
token.expose_secret(),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| DatabaseError::Pool(e.to_string()))?
|
||||
} else {
|
||||
libsql_backend::LibSqlBackend::new_local(db_path)
|
||||
.await
|
||||
.map_err(|e| DatabaseError::Pool(e.to_string()))?
|
||||
};
|
||||
backend.run_migrations().await?;
|
||||
Ok(Arc::new(backend))
|
||||
}
|
||||
#[cfg(feature = "postgres")]
|
||||
_ => {
|
||||
let pg = postgres::PgBackend::new(config)
|
||||
.await
|
||||
.map_err(|e| DatabaseError::Pool(e.to_string()))?;
|
||||
pg.run_migrations().await?;
|
||||
Ok(Arc::new(pg))
|
||||
}
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
_ => Err(DatabaseError::Pool(
|
||||
"No database backend available. Enable 'postgres' or 'libsql' feature.".to_string(),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Backend-agnostic database trait.
|
||||
///
|
||||
/// Combines all persistence operations from Store, Repository, and related
|
||||
/// stores into a single trait that can be implemented for different backends.
|
||||
#[async_trait]
|
||||
pub trait Database: Send + Sync {
|
||||
/// Run schema migrations for this backend.
|
||||
async fn run_migrations(&self) -> Result<(), DatabaseError>;
|
||||
|
||||
// ==================== Conversations ====================
|
||||
|
||||
/// Create a new conversation.
|
||||
async fn create_conversation(
|
||||
&self,
|
||||
channel: &str,
|
||||
user_id: &str,
|
||||
thread_id: Option<&str>,
|
||||
) -> Result<Uuid, DatabaseError>;
|
||||
|
||||
/// Update conversation last activity.
|
||||
async fn touch_conversation(&self, id: Uuid) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Add a message to a conversation.
|
||||
async fn add_conversation_message(
|
||||
&self,
|
||||
conversation_id: Uuid,
|
||||
role: &str,
|
||||
content: &str,
|
||||
) -> Result<Uuid, DatabaseError>;
|
||||
|
||||
/// Ensure a conversation row exists (upsert).
|
||||
async fn ensure_conversation(
|
||||
&self,
|
||||
id: Uuid,
|
||||
channel: &str,
|
||||
user_id: &str,
|
||||
thread_id: Option<&str>,
|
||||
) -> Result<(), DatabaseError>;
|
||||
|
||||
/// List conversations with a title preview.
|
||||
async fn list_conversations_with_preview(
|
||||
&self,
|
||||
user_id: &str,
|
||||
channel: &str,
|
||||
limit: i64,
|
||||
) -> Result<Vec<ConversationSummary>, DatabaseError>;
|
||||
|
||||
/// Get or create the singleton assistant conversation.
|
||||
async fn get_or_create_assistant_conversation(
|
||||
&self,
|
||||
user_id: &str,
|
||||
channel: &str,
|
||||
) -> Result<Uuid, DatabaseError>;
|
||||
|
||||
/// Create a conversation with specific metadata.
|
||||
async fn create_conversation_with_metadata(
|
||||
&self,
|
||||
channel: &str,
|
||||
user_id: &str,
|
||||
metadata: &serde_json::Value,
|
||||
) -> Result<Uuid, DatabaseError>;
|
||||
|
||||
/// Load messages with cursor-based pagination.
|
||||
async fn list_conversation_messages_paginated(
|
||||
&self,
|
||||
conversation_id: Uuid,
|
||||
before: Option<DateTime<Utc>>,
|
||||
limit: i64,
|
||||
) -> Result<(Vec<ConversationMessage>, bool), DatabaseError>;
|
||||
|
||||
/// Merge a single key into conversation metadata.
|
||||
async fn update_conversation_metadata_field(
|
||||
&self,
|
||||
id: Uuid,
|
||||
key: &str,
|
||||
value: &serde_json::Value,
|
||||
) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Read conversation metadata.
|
||||
async fn get_conversation_metadata(
|
||||
&self,
|
||||
id: Uuid,
|
||||
) -> Result<Option<serde_json::Value>, DatabaseError>;
|
||||
|
||||
/// Load all messages for a conversation.
|
||||
async fn list_conversation_messages(
|
||||
&self,
|
||||
conversation_id: Uuid,
|
||||
) -> Result<Vec<ConversationMessage>, DatabaseError>;
|
||||
|
||||
/// Check if a conversation belongs to a specific user.
|
||||
async fn conversation_belongs_to_user(
|
||||
&self,
|
||||
conversation_id: Uuid,
|
||||
user_id: &str,
|
||||
) -> Result<bool, DatabaseError>;
|
||||
|
||||
// ==================== Jobs ====================
|
||||
|
||||
/// Save a job context.
|
||||
async fn save_job(&self, ctx: &JobContext) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Get a job by ID.
|
||||
async fn get_job(&self, id: Uuid) -> Result<Option<JobContext>, DatabaseError>;
|
||||
|
||||
/// Update job status.
|
||||
async fn update_job_status(
|
||||
&self,
|
||||
id: Uuid,
|
||||
status: JobState,
|
||||
failure_reason: Option<&str>,
|
||||
) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Mark job as stuck.
|
||||
async fn mark_job_stuck(&self, id: Uuid) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Get stuck jobs.
|
||||
async fn get_stuck_jobs(&self) -> Result<Vec<Uuid>, DatabaseError>;
|
||||
|
||||
// ==================== Actions ====================
|
||||
|
||||
/// Save a job action.
|
||||
async fn save_action(&self, job_id: Uuid, action: &ActionRecord) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Get actions for a job.
|
||||
async fn get_job_actions(&self, job_id: Uuid) -> Result<Vec<ActionRecord>, DatabaseError>;
|
||||
|
||||
// ==================== LLM Calls ====================
|
||||
|
||||
/// Record an LLM call.
|
||||
async fn record_llm_call(&self, record: &LlmCallRecord<'_>) -> Result<Uuid, DatabaseError>;
|
||||
|
||||
// ==================== Estimation Snapshots ====================
|
||||
|
||||
/// Save an estimation snapshot.
|
||||
async fn save_estimation_snapshot(
|
||||
&self,
|
||||
job_id: Uuid,
|
||||
category: &str,
|
||||
tool_names: &[String],
|
||||
estimated_cost: Decimal,
|
||||
estimated_time_secs: i32,
|
||||
estimated_value: Decimal,
|
||||
) -> Result<Uuid, DatabaseError>;
|
||||
|
||||
/// Update estimation snapshot with actual values.
|
||||
async fn update_estimation_actuals(
|
||||
&self,
|
||||
id: Uuid,
|
||||
actual_cost: Decimal,
|
||||
actual_time_secs: i32,
|
||||
actual_value: Option<Decimal>,
|
||||
) -> Result<(), DatabaseError>;
|
||||
|
||||
// ==================== Sandbox Jobs ====================
|
||||
|
||||
/// Insert a new sandbox job.
|
||||
async fn save_sandbox_job(&self, job: &SandboxJobRecord) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Get a sandbox job by ID.
|
||||
async fn get_sandbox_job(&self, id: Uuid) -> Result<Option<SandboxJobRecord>, DatabaseError>;
|
||||
|
||||
/// List all sandbox jobs, most recent first.
|
||||
async fn list_sandbox_jobs(&self) -> Result<Vec<SandboxJobRecord>, DatabaseError>;
|
||||
|
||||
/// Update sandbox job status.
|
||||
async fn update_sandbox_job_status(
|
||||
&self,
|
||||
id: Uuid,
|
||||
status: &str,
|
||||
success: Option<bool>,
|
||||
message: Option<&str>,
|
||||
started_at: Option<DateTime<Utc>>,
|
||||
completed_at: Option<DateTime<Utc>>,
|
||||
) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Mark stale sandbox jobs as interrupted.
|
||||
async fn cleanup_stale_sandbox_jobs(&self) -> Result<u64, DatabaseError>;
|
||||
|
||||
/// Get sandbox job summary.
|
||||
async fn sandbox_job_summary(&self) -> Result<SandboxJobSummary, DatabaseError>;
|
||||
|
||||
/// List sandbox jobs for a specific user, most recent first.
|
||||
async fn list_sandbox_jobs_for_user(
|
||||
&self,
|
||||
user_id: &str,
|
||||
) -> Result<Vec<SandboxJobRecord>, DatabaseError>;
|
||||
|
||||
/// Get sandbox job summary for a specific user.
|
||||
async fn sandbox_job_summary_for_user(
|
||||
&self,
|
||||
user_id: &str,
|
||||
) -> Result<SandboxJobSummary, DatabaseError>;
|
||||
|
||||
/// Check if a sandbox job belongs to a specific user.
|
||||
async fn sandbox_job_belongs_to_user(
|
||||
&self,
|
||||
job_id: Uuid,
|
||||
user_id: &str,
|
||||
) -> Result<bool, DatabaseError>;
|
||||
|
||||
/// Update sandbox job mode.
|
||||
async fn update_sandbox_job_mode(&self, id: Uuid, mode: &str) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Get sandbox job mode.
|
||||
async fn get_sandbox_job_mode(&self, id: Uuid) -> Result<Option<String>, DatabaseError>;
|
||||
|
||||
// ==================== Job Events ====================
|
||||
|
||||
/// Persist a job event.
|
||||
async fn save_job_event(
|
||||
&self,
|
||||
job_id: Uuid,
|
||||
event_type: &str,
|
||||
data: &serde_json::Value,
|
||||
) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Load all job events.
|
||||
async fn list_job_events(&self, job_id: Uuid) -> Result<Vec<JobEventRecord>, DatabaseError>;
|
||||
|
||||
// ==================== Routines ====================
|
||||
|
||||
/// Create a new routine.
|
||||
async fn create_routine(&self, routine: &Routine) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Get a routine by ID.
|
||||
async fn get_routine(&self, id: Uuid) -> Result<Option<Routine>, DatabaseError>;
|
||||
|
||||
/// Get a routine by user_id and name.
|
||||
async fn get_routine_by_name(
|
||||
&self,
|
||||
user_id: &str,
|
||||
name: &str,
|
||||
) -> Result<Option<Routine>, DatabaseError>;
|
||||
|
||||
/// List routines for a user.
|
||||
async fn list_routines(&self, user_id: &str) -> Result<Vec<Routine>, DatabaseError>;
|
||||
|
||||
/// List all enabled event routines.
|
||||
async fn list_event_routines(&self) -> Result<Vec<Routine>, DatabaseError>;
|
||||
|
||||
/// List due cron routines.
|
||||
async fn list_due_cron_routines(&self) -> Result<Vec<Routine>, DatabaseError>;
|
||||
|
||||
/// Update a routine.
|
||||
async fn update_routine(&self, routine: &Routine) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Update runtime state after a routine fires.
|
||||
async fn update_routine_runtime(
|
||||
&self,
|
||||
id: Uuid,
|
||||
last_run_at: DateTime<Utc>,
|
||||
next_fire_at: Option<DateTime<Utc>>,
|
||||
run_count: u64,
|
||||
consecutive_failures: u32,
|
||||
state: &serde_json::Value,
|
||||
) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Delete a routine.
|
||||
async fn delete_routine(&self, id: Uuid) -> Result<bool, DatabaseError>;
|
||||
|
||||
// ==================== Routine Runs ====================
|
||||
|
||||
/// Record a routine run starting.
|
||||
async fn create_routine_run(&self, run: &RoutineRun) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Complete a routine run.
|
||||
async fn complete_routine_run(
|
||||
&self,
|
||||
id: Uuid,
|
||||
status: RunStatus,
|
||||
result_summary: Option<&str>,
|
||||
tokens_used: Option<i32>,
|
||||
) -> Result<(), DatabaseError>;
|
||||
|
||||
/// List recent runs for a routine.
|
||||
async fn list_routine_runs(
|
||||
&self,
|
||||
routine_id: Uuid,
|
||||
limit: i64,
|
||||
) -> Result<Vec<RoutineRun>, DatabaseError>;
|
||||
|
||||
/// Count currently running runs for a routine.
|
||||
async fn count_running_routine_runs(&self, routine_id: Uuid) -> Result<i64, DatabaseError>;
|
||||
|
||||
// ==================== Tool Failures ====================
|
||||
|
||||
/// Record a tool failure (upsert).
|
||||
async fn record_tool_failure(
|
||||
&self,
|
||||
tool_name: &str,
|
||||
error_message: &str,
|
||||
) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Get broken tools exceeding threshold.
|
||||
async fn get_broken_tools(&self, threshold: i32) -> Result<Vec<BrokenTool>, DatabaseError>;
|
||||
|
||||
/// Mark a tool as repaired.
|
||||
async fn mark_tool_repaired(&self, tool_name: &str) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Increment repair attempts.
|
||||
async fn increment_repair_attempts(&self, tool_name: &str) -> Result<(), DatabaseError>;
|
||||
|
||||
// ==================== Settings ====================
|
||||
|
||||
/// Get a single setting.
|
||||
async fn get_setting(
|
||||
&self,
|
||||
user_id: &str,
|
||||
key: &str,
|
||||
) -> Result<Option<serde_json::Value>, DatabaseError>;
|
||||
|
||||
/// Get a single setting with metadata.
|
||||
async fn get_setting_full(
|
||||
&self,
|
||||
user_id: &str,
|
||||
key: &str,
|
||||
) -> Result<Option<SettingRow>, DatabaseError>;
|
||||
|
||||
/// Set a single setting (upsert).
|
||||
async fn set_setting(
|
||||
&self,
|
||||
user_id: &str,
|
||||
key: &str,
|
||||
value: &serde_json::Value,
|
||||
) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Delete a single setting.
|
||||
async fn delete_setting(&self, user_id: &str, key: &str) -> Result<bool, DatabaseError>;
|
||||
|
||||
/// List all settings for a user.
|
||||
async fn list_settings(&self, user_id: &str) -> Result<Vec<SettingRow>, DatabaseError>;
|
||||
|
||||
/// Get all settings as a flat map.
|
||||
async fn get_all_settings(
|
||||
&self,
|
||||
user_id: &str,
|
||||
) -> Result<HashMap<String, serde_json::Value>, DatabaseError>;
|
||||
|
||||
/// Bulk-write settings atomically.
|
||||
async fn set_all_settings(
|
||||
&self,
|
||||
user_id: &str,
|
||||
settings: &HashMap<String, serde_json::Value>,
|
||||
) -> Result<(), DatabaseError>;
|
||||
|
||||
/// Check if settings exist for a user.
|
||||
async fn has_settings(&self, user_id: &str) -> Result<bool, DatabaseError>;
|
||||
|
||||
// ==================== Workspace: Documents ====================
|
||||
|
||||
/// Get a document by path.
|
||||
async fn get_document_by_path(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
path: &str,
|
||||
) -> Result<MemoryDocument, WorkspaceError>;
|
||||
|
||||
/// Get a document by ID.
|
||||
async fn get_document_by_id(&self, id: Uuid) -> Result<MemoryDocument, WorkspaceError>;
|
||||
|
||||
/// Get or create a document by path.
|
||||
async fn get_or_create_document_by_path(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
path: &str,
|
||||
) -> Result<MemoryDocument, WorkspaceError>;
|
||||
|
||||
/// Update a document's content.
|
||||
async fn update_document(&self, id: Uuid, content: &str) -> Result<(), WorkspaceError>;
|
||||
|
||||
/// Delete a document by path.
|
||||
async fn delete_document_by_path(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
path: &str,
|
||||
) -> Result<(), WorkspaceError>;
|
||||
|
||||
/// List files and directories in a directory path.
|
||||
async fn list_directory(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
directory: &str,
|
||||
) -> Result<Vec<WorkspaceEntry>, WorkspaceError>;
|
||||
|
||||
/// List all file paths in the workspace.
|
||||
async fn list_all_paths(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
) -> Result<Vec<String>, WorkspaceError>;
|
||||
|
||||
/// List all documents for a user.
|
||||
async fn list_documents(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
) -> Result<Vec<MemoryDocument>, WorkspaceError>;
|
||||
|
||||
// ==================== Workspace: Chunks ====================
|
||||
|
||||
/// Delete all chunks for a document.
|
||||
async fn delete_chunks(&self, document_id: Uuid) -> Result<(), WorkspaceError>;
|
||||
|
||||
/// Insert a chunk.
|
||||
async fn insert_chunk(
|
||||
&self,
|
||||
document_id: Uuid,
|
||||
chunk_index: i32,
|
||||
content: &str,
|
||||
embedding: Option<&[f32]>,
|
||||
) -> Result<Uuid, WorkspaceError>;
|
||||
|
||||
/// Update a chunk's embedding.
|
||||
async fn update_chunk_embedding(
|
||||
&self,
|
||||
chunk_id: Uuid,
|
||||
embedding: &[f32],
|
||||
) -> Result<(), WorkspaceError>;
|
||||
|
||||
/// Get chunks without embeddings for backfilling.
|
||||
async fn get_chunks_without_embeddings(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
limit: usize,
|
||||
) -> Result<Vec<MemoryChunk>, WorkspaceError>;
|
||||
|
||||
// ==================== Workspace: Search ====================
|
||||
|
||||
/// Perform hybrid search combining FTS and vector similarity.
|
||||
async fn hybrid_search(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
query: &str,
|
||||
embedding: Option<&[f32]>,
|
||||
config: &SearchConfig,
|
||||
) -> Result<Vec<SearchResult>, WorkspaceError>;
|
||||
}
|
||||
@@ -0,0 +1,627 @@
|
||||
//! PostgreSQL backend for the Database trait.
|
||||
//!
|
||||
//! Delegates to the existing `Store` (history) and `Repository` (workspace)
|
||||
//! implementations, avoiding SQL duplication.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
use deadpool_postgres::Pool;
|
||||
use rust_decimal::Decimal;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::agent::BrokenTool;
|
||||
use crate::agent::routine::{Routine, RoutineRun, RunStatus};
|
||||
use crate::config::DatabaseConfig;
|
||||
use crate::context::{ActionRecord, JobContext, JobState};
|
||||
use crate::db::Database;
|
||||
use crate::error::{DatabaseError, WorkspaceError};
|
||||
use crate::history::{
|
||||
ConversationMessage, ConversationSummary, JobEventRecord, LlmCallRecord, SandboxJobRecord,
|
||||
SandboxJobSummary, SettingRow, Store,
|
||||
};
|
||||
use crate::workspace::{
|
||||
MemoryChunk, MemoryDocument, Repository, SearchConfig, SearchResult, WorkspaceEntry,
|
||||
};
|
||||
|
||||
/// PostgreSQL database backend.
|
||||
///
|
||||
/// Wraps the existing `Store` (for history/conversations/jobs/routines/settings)
|
||||
/// and `Repository` (for workspace documents/chunks/search) to implement the
|
||||
/// unified `Database` trait.
|
||||
pub struct PgBackend {
|
||||
store: Store,
|
||||
repo: Repository,
|
||||
}
|
||||
|
||||
impl PgBackend {
|
||||
/// Create a new PostgreSQL backend from configuration.
|
||||
pub async fn new(config: &DatabaseConfig) -> Result<Self, DatabaseError> {
|
||||
let store = Store::new(config).await?;
|
||||
let repo = Repository::new(store.pool());
|
||||
Ok(Self { store, repo })
|
||||
}
|
||||
|
||||
/// Get a clone of the connection pool.
|
||||
///
|
||||
/// Useful for sharing with components that still need raw pool access.
|
||||
pub fn pool(&self) -> Pool {
|
||||
self.store.pool()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Database for PgBackend {
|
||||
async fn run_migrations(&self) -> Result<(), DatabaseError> {
|
||||
self.store.run_migrations().await
|
||||
}
|
||||
|
||||
// ==================== Conversations ====================
|
||||
|
||||
async fn create_conversation(
|
||||
&self,
|
||||
channel: &str,
|
||||
user_id: &str,
|
||||
thread_id: Option<&str>,
|
||||
) -> Result<Uuid, DatabaseError> {
|
||||
self.store
|
||||
.create_conversation(channel, user_id, thread_id)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn touch_conversation(&self, id: Uuid) -> Result<(), DatabaseError> {
|
||||
self.store.touch_conversation(id).await
|
||||
}
|
||||
|
||||
async fn add_conversation_message(
|
||||
&self,
|
||||
conversation_id: Uuid,
|
||||
role: &str,
|
||||
content: &str,
|
||||
) -> Result<Uuid, DatabaseError> {
|
||||
self.store
|
||||
.add_conversation_message(conversation_id, role, content)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn ensure_conversation(
|
||||
&self,
|
||||
id: Uuid,
|
||||
channel: &str,
|
||||
user_id: &str,
|
||||
thread_id: Option<&str>,
|
||||
) -> Result<(), DatabaseError> {
|
||||
self.store
|
||||
.ensure_conversation(id, channel, user_id, thread_id)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_conversations_with_preview(
|
||||
&self,
|
||||
user_id: &str,
|
||||
channel: &str,
|
||||
limit: i64,
|
||||
) -> Result<Vec<ConversationSummary>, DatabaseError> {
|
||||
self.store
|
||||
.list_conversations_with_preview(user_id, channel, limit)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_or_create_assistant_conversation(
|
||||
&self,
|
||||
user_id: &str,
|
||||
channel: &str,
|
||||
) -> Result<Uuid, DatabaseError> {
|
||||
self.store
|
||||
.get_or_create_assistant_conversation(user_id, channel)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn create_conversation_with_metadata(
|
||||
&self,
|
||||
channel: &str,
|
||||
user_id: &str,
|
||||
metadata: &serde_json::Value,
|
||||
) -> Result<Uuid, DatabaseError> {
|
||||
self.store
|
||||
.create_conversation_with_metadata(channel, user_id, metadata)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_conversation_messages_paginated(
|
||||
&self,
|
||||
conversation_id: Uuid,
|
||||
before: Option<DateTime<Utc>>,
|
||||
limit: i64,
|
||||
) -> Result<(Vec<ConversationMessage>, bool), DatabaseError> {
|
||||
self.store
|
||||
.list_conversation_messages_paginated(conversation_id, before, limit)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn update_conversation_metadata_field(
|
||||
&self,
|
||||
id: Uuid,
|
||||
key: &str,
|
||||
value: &serde_json::Value,
|
||||
) -> Result<(), DatabaseError> {
|
||||
self.store
|
||||
.update_conversation_metadata_field(id, key, value)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_conversation_metadata(
|
||||
&self,
|
||||
id: Uuid,
|
||||
) -> Result<Option<serde_json::Value>, DatabaseError> {
|
||||
self.store.get_conversation_metadata(id).await
|
||||
}
|
||||
|
||||
async fn list_conversation_messages(
|
||||
&self,
|
||||
conversation_id: Uuid,
|
||||
) -> Result<Vec<ConversationMessage>, DatabaseError> {
|
||||
self.store.list_conversation_messages(conversation_id).await
|
||||
}
|
||||
|
||||
async fn conversation_belongs_to_user(
|
||||
&self,
|
||||
conversation_id: Uuid,
|
||||
user_id: &str,
|
||||
) -> Result<bool, DatabaseError> {
|
||||
self.store
|
||||
.conversation_belongs_to_user(conversation_id, user_id)
|
||||
.await
|
||||
}
|
||||
|
||||
// ==================== Jobs ====================
|
||||
|
||||
async fn save_job(&self, ctx: &JobContext) -> Result<(), DatabaseError> {
|
||||
self.store.save_job(ctx).await
|
||||
}
|
||||
|
||||
async fn get_job(&self, id: Uuid) -> Result<Option<JobContext>, DatabaseError> {
|
||||
self.store.get_job(id).await
|
||||
}
|
||||
|
||||
async fn update_job_status(
|
||||
&self,
|
||||
id: Uuid,
|
||||
status: JobState,
|
||||
failure_reason: Option<&str>,
|
||||
) -> Result<(), DatabaseError> {
|
||||
self.store
|
||||
.update_job_status(id, status, failure_reason)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn mark_job_stuck(&self, id: Uuid) -> Result<(), DatabaseError> {
|
||||
self.store.mark_job_stuck(id).await
|
||||
}
|
||||
|
||||
async fn get_stuck_jobs(&self) -> Result<Vec<Uuid>, DatabaseError> {
|
||||
self.store.get_stuck_jobs().await
|
||||
}
|
||||
|
||||
// ==================== Actions ====================
|
||||
|
||||
async fn save_action(&self, job_id: Uuid, action: &ActionRecord) -> Result<(), DatabaseError> {
|
||||
self.store.save_action(job_id, action).await
|
||||
}
|
||||
|
||||
async fn get_job_actions(&self, job_id: Uuid) -> Result<Vec<ActionRecord>, DatabaseError> {
|
||||
self.store.get_job_actions(job_id).await
|
||||
}
|
||||
|
||||
// ==================== LLM Calls ====================
|
||||
|
||||
async fn record_llm_call(&self, record: &LlmCallRecord<'_>) -> Result<Uuid, DatabaseError> {
|
||||
self.store.record_llm_call(record).await
|
||||
}
|
||||
|
||||
// ==================== Estimation Snapshots ====================
|
||||
|
||||
async fn save_estimation_snapshot(
|
||||
&self,
|
||||
job_id: Uuid,
|
||||
category: &str,
|
||||
tool_names: &[String],
|
||||
estimated_cost: Decimal,
|
||||
estimated_time_secs: i32,
|
||||
estimated_value: Decimal,
|
||||
) -> Result<Uuid, DatabaseError> {
|
||||
self.store
|
||||
.save_estimation_snapshot(
|
||||
job_id,
|
||||
category,
|
||||
tool_names,
|
||||
estimated_cost,
|
||||
estimated_time_secs,
|
||||
estimated_value,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn update_estimation_actuals(
|
||||
&self,
|
||||
id: Uuid,
|
||||
actual_cost: Decimal,
|
||||
actual_time_secs: i32,
|
||||
actual_value: Option<Decimal>,
|
||||
) -> Result<(), DatabaseError> {
|
||||
self.store
|
||||
.update_estimation_actuals(id, actual_cost, actual_time_secs, actual_value)
|
||||
.await
|
||||
}
|
||||
|
||||
// ==================== Sandbox Jobs ====================
|
||||
|
||||
async fn save_sandbox_job(&self, job: &SandboxJobRecord) -> Result<(), DatabaseError> {
|
||||
self.store.save_sandbox_job(job).await
|
||||
}
|
||||
|
||||
async fn get_sandbox_job(&self, id: Uuid) -> Result<Option<SandboxJobRecord>, DatabaseError> {
|
||||
self.store.get_sandbox_job(id).await
|
||||
}
|
||||
|
||||
async fn list_sandbox_jobs(&self) -> Result<Vec<SandboxJobRecord>, DatabaseError> {
|
||||
self.store.list_sandbox_jobs().await
|
||||
}
|
||||
|
||||
async fn update_sandbox_job_status(
|
||||
&self,
|
||||
id: Uuid,
|
||||
status: &str,
|
||||
success: Option<bool>,
|
||||
message: Option<&str>,
|
||||
started_at: Option<DateTime<Utc>>,
|
||||
completed_at: Option<DateTime<Utc>>,
|
||||
) -> Result<(), DatabaseError> {
|
||||
self.store
|
||||
.update_sandbox_job_status(id, status, success, message, started_at, completed_at)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn cleanup_stale_sandbox_jobs(&self) -> Result<u64, DatabaseError> {
|
||||
self.store.cleanup_stale_sandbox_jobs().await
|
||||
}
|
||||
|
||||
async fn sandbox_job_summary(&self) -> Result<SandboxJobSummary, DatabaseError> {
|
||||
self.store.sandbox_job_summary().await
|
||||
}
|
||||
|
||||
async fn list_sandbox_jobs_for_user(
|
||||
&self,
|
||||
user_id: &str,
|
||||
) -> Result<Vec<SandboxJobRecord>, DatabaseError> {
|
||||
self.store.list_sandbox_jobs_for_user(user_id).await
|
||||
}
|
||||
|
||||
async fn sandbox_job_summary_for_user(
|
||||
&self,
|
||||
user_id: &str,
|
||||
) -> Result<SandboxJobSummary, DatabaseError> {
|
||||
self.store.sandbox_job_summary_for_user(user_id).await
|
||||
}
|
||||
|
||||
async fn sandbox_job_belongs_to_user(
|
||||
&self,
|
||||
job_id: Uuid,
|
||||
user_id: &str,
|
||||
) -> Result<bool, DatabaseError> {
|
||||
self.store
|
||||
.sandbox_job_belongs_to_user(job_id, user_id)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn update_sandbox_job_mode(&self, id: Uuid, mode: &str) -> Result<(), DatabaseError> {
|
||||
self.store.update_sandbox_job_mode(id, mode).await
|
||||
}
|
||||
|
||||
async fn get_sandbox_job_mode(&self, id: Uuid) -> Result<Option<String>, DatabaseError> {
|
||||
self.store.get_sandbox_job_mode(id).await
|
||||
}
|
||||
|
||||
// ==================== Job Events ====================
|
||||
|
||||
async fn save_job_event(
|
||||
&self,
|
||||
job_id: Uuid,
|
||||
event_type: &str,
|
||||
data: &serde_json::Value,
|
||||
) -> Result<(), DatabaseError> {
|
||||
self.store.save_job_event(job_id, event_type, data).await
|
||||
}
|
||||
|
||||
async fn list_job_events(&self, job_id: Uuid) -> Result<Vec<JobEventRecord>, DatabaseError> {
|
||||
self.store.list_job_events(job_id).await
|
||||
}
|
||||
|
||||
// ==================== Routines ====================
|
||||
|
||||
async fn create_routine(&self, routine: &Routine) -> Result<(), DatabaseError> {
|
||||
self.store.create_routine(routine).await
|
||||
}
|
||||
|
||||
async fn get_routine(&self, id: Uuid) -> Result<Option<Routine>, DatabaseError> {
|
||||
self.store.get_routine(id).await
|
||||
}
|
||||
|
||||
async fn get_routine_by_name(
|
||||
&self,
|
||||
user_id: &str,
|
||||
name: &str,
|
||||
) -> Result<Option<Routine>, DatabaseError> {
|
||||
self.store.get_routine_by_name(user_id, name).await
|
||||
}
|
||||
|
||||
async fn list_routines(&self, user_id: &str) -> Result<Vec<Routine>, DatabaseError> {
|
||||
self.store.list_routines(user_id).await
|
||||
}
|
||||
|
||||
async fn list_event_routines(&self) -> Result<Vec<Routine>, DatabaseError> {
|
||||
self.store.list_event_routines().await
|
||||
}
|
||||
|
||||
async fn list_due_cron_routines(&self) -> Result<Vec<Routine>, DatabaseError> {
|
||||
self.store.list_due_cron_routines().await
|
||||
}
|
||||
|
||||
async fn update_routine(&self, routine: &Routine) -> Result<(), DatabaseError> {
|
||||
self.store.update_routine(routine).await
|
||||
}
|
||||
|
||||
async fn update_routine_runtime(
|
||||
&self,
|
||||
id: Uuid,
|
||||
last_run_at: DateTime<Utc>,
|
||||
next_fire_at: Option<DateTime<Utc>>,
|
||||
run_count: u64,
|
||||
consecutive_failures: u32,
|
||||
state: &serde_json::Value,
|
||||
) -> Result<(), DatabaseError> {
|
||||
self.store
|
||||
.update_routine_runtime(
|
||||
id,
|
||||
last_run_at,
|
||||
next_fire_at,
|
||||
run_count,
|
||||
consecutive_failures,
|
||||
state,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_routine(&self, id: Uuid) -> Result<bool, DatabaseError> {
|
||||
self.store.delete_routine(id).await
|
||||
}
|
||||
|
||||
// ==================== Routine Runs ====================
|
||||
|
||||
async fn create_routine_run(&self, run: &RoutineRun) -> Result<(), DatabaseError> {
|
||||
self.store.create_routine_run(run).await
|
||||
}
|
||||
|
||||
async fn complete_routine_run(
|
||||
&self,
|
||||
id: Uuid,
|
||||
status: RunStatus,
|
||||
result_summary: Option<&str>,
|
||||
tokens_used: Option<i32>,
|
||||
) -> Result<(), DatabaseError> {
|
||||
self.store
|
||||
.complete_routine_run(id, status, result_summary, tokens_used)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_routine_runs(
|
||||
&self,
|
||||
routine_id: Uuid,
|
||||
limit: i64,
|
||||
) -> Result<Vec<RoutineRun>, DatabaseError> {
|
||||
self.store.list_routine_runs(routine_id, limit).await
|
||||
}
|
||||
|
||||
async fn count_running_routine_runs(&self, routine_id: Uuid) -> Result<i64, DatabaseError> {
|
||||
self.store.count_running_routine_runs(routine_id).await
|
||||
}
|
||||
|
||||
// ==================== Tool Failures ====================
|
||||
|
||||
async fn record_tool_failure(
|
||||
&self,
|
||||
tool_name: &str,
|
||||
error_message: &str,
|
||||
) -> Result<(), DatabaseError> {
|
||||
self.store
|
||||
.record_tool_failure(tool_name, error_message)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_broken_tools(&self, threshold: i32) -> Result<Vec<BrokenTool>, DatabaseError> {
|
||||
self.store.get_broken_tools(threshold).await
|
||||
}
|
||||
|
||||
async fn mark_tool_repaired(&self, tool_name: &str) -> Result<(), DatabaseError> {
|
||||
self.store.mark_tool_repaired(tool_name).await
|
||||
}
|
||||
|
||||
async fn increment_repair_attempts(&self, tool_name: &str) -> Result<(), DatabaseError> {
|
||||
self.store.increment_repair_attempts(tool_name).await
|
||||
}
|
||||
|
||||
// ==================== Settings ====================
|
||||
|
||||
async fn get_setting(
|
||||
&self,
|
||||
user_id: &str,
|
||||
key: &str,
|
||||
) -> Result<Option<serde_json::Value>, DatabaseError> {
|
||||
self.store.get_setting(user_id, key).await
|
||||
}
|
||||
|
||||
async fn get_setting_full(
|
||||
&self,
|
||||
user_id: &str,
|
||||
key: &str,
|
||||
) -> Result<Option<SettingRow>, DatabaseError> {
|
||||
self.store.get_setting_full(user_id, key).await
|
||||
}
|
||||
|
||||
async fn set_setting(
|
||||
&self,
|
||||
user_id: &str,
|
||||
key: &str,
|
||||
value: &serde_json::Value,
|
||||
) -> Result<(), DatabaseError> {
|
||||
self.store.set_setting(user_id, key, value).await
|
||||
}
|
||||
|
||||
async fn delete_setting(&self, user_id: &str, key: &str) -> Result<bool, DatabaseError> {
|
||||
self.store.delete_setting(user_id, key).await
|
||||
}
|
||||
|
||||
async fn list_settings(&self, user_id: &str) -> Result<Vec<SettingRow>, DatabaseError> {
|
||||
self.store.list_settings(user_id).await
|
||||
}
|
||||
|
||||
async fn get_all_settings(
|
||||
&self,
|
||||
user_id: &str,
|
||||
) -> Result<HashMap<String, serde_json::Value>, DatabaseError> {
|
||||
self.store.get_all_settings(user_id).await
|
||||
}
|
||||
|
||||
async fn set_all_settings(
|
||||
&self,
|
||||
user_id: &str,
|
||||
settings: &HashMap<String, serde_json::Value>,
|
||||
) -> Result<(), DatabaseError> {
|
||||
self.store.set_all_settings(user_id, settings).await
|
||||
}
|
||||
|
||||
async fn has_settings(&self, user_id: &str) -> Result<bool, DatabaseError> {
|
||||
self.store.has_settings(user_id).await
|
||||
}
|
||||
|
||||
// ==================== Workspace: Documents ====================
|
||||
|
||||
async fn get_document_by_path(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
path: &str,
|
||||
) -> Result<MemoryDocument, WorkspaceError> {
|
||||
self.repo
|
||||
.get_document_by_path(user_id, agent_id, path)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn get_document_by_id(&self, id: Uuid) -> Result<MemoryDocument, WorkspaceError> {
|
||||
self.repo.get_document_by_id(id).await
|
||||
}
|
||||
|
||||
async fn get_or_create_document_by_path(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
path: &str,
|
||||
) -> Result<MemoryDocument, WorkspaceError> {
|
||||
self.repo
|
||||
.get_or_create_document_by_path(user_id, agent_id, path)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn update_document(&self, id: Uuid, content: &str) -> Result<(), WorkspaceError> {
|
||||
self.repo.update_document(id, content).await
|
||||
}
|
||||
|
||||
async fn delete_document_by_path(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
path: &str,
|
||||
) -> Result<(), WorkspaceError> {
|
||||
self.repo
|
||||
.delete_document_by_path(user_id, agent_id, path)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn list_directory(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
directory: &str,
|
||||
) -> Result<Vec<WorkspaceEntry>, WorkspaceError> {
|
||||
self.repo.list_directory(user_id, agent_id, directory).await
|
||||
}
|
||||
|
||||
async fn list_all_paths(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
) -> Result<Vec<String>, WorkspaceError> {
|
||||
self.repo.list_all_paths(user_id, agent_id).await
|
||||
}
|
||||
|
||||
async fn list_documents(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
) -> Result<Vec<MemoryDocument>, WorkspaceError> {
|
||||
self.repo.list_documents(user_id, agent_id).await
|
||||
}
|
||||
|
||||
// ==================== Workspace: Chunks ====================
|
||||
|
||||
async fn delete_chunks(&self, document_id: Uuid) -> Result<(), WorkspaceError> {
|
||||
self.repo.delete_chunks(document_id).await
|
||||
}
|
||||
|
||||
async fn insert_chunk(
|
||||
&self,
|
||||
document_id: Uuid,
|
||||
chunk_index: i32,
|
||||
content: &str,
|
||||
embedding: Option<&[f32]>,
|
||||
) -> Result<Uuid, WorkspaceError> {
|
||||
self.repo
|
||||
.insert_chunk(document_id, chunk_index, content, embedding)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn update_chunk_embedding(
|
||||
&self,
|
||||
chunk_id: Uuid,
|
||||
embedding: &[f32],
|
||||
) -> Result<(), WorkspaceError> {
|
||||
self.repo.update_chunk_embedding(chunk_id, embedding).await
|
||||
}
|
||||
|
||||
async fn get_chunks_without_embeddings(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
limit: usize,
|
||||
) -> Result<Vec<MemoryChunk>, WorkspaceError> {
|
||||
self.repo
|
||||
.get_chunks_without_embeddings(user_id, agent_id, limit)
|
||||
.await
|
||||
}
|
||||
|
||||
// ==================== Workspace: Search ====================
|
||||
|
||||
async fn hybrid_search(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
query: &str,
|
||||
embedding: Option<&[f32]>,
|
||||
config: &SearchConfig,
|
||||
) -> Result<Vec<SearchResult>, WorkspaceError> {
|
||||
self.repo
|
||||
.hybrid_search(user_id, agent_id, query, embedding, config)
|
||||
.await
|
||||
}
|
||||
}
|
||||
@@ -87,14 +87,21 @@ pub enum DatabaseError {
|
||||
#[error("Serialization error: {0}")]
|
||||
Serialization(String),
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
#[error("PostgreSQL error: {0}")]
|
||||
Postgres(#[from] tokio_postgres::Error),
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
#[error("Pool build error: {0}")]
|
||||
PoolBuild(#[from] deadpool_postgres::BuildError),
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
#[error("Pool runtime error: {0}")]
|
||||
PoolRuntime(#[from] deadpool_postgres::PoolError),
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
#[error("LibSQL error: {0}")]
|
||||
LibSql(#[from] libsql::Error),
|
||||
}
|
||||
|
||||
/// Channel-related errors.
|
||||
|
||||
@@ -57,7 +57,7 @@ pub struct ExtensionManager {
|
||||
_tunnel_url: Option<String>,
|
||||
user_id: String,
|
||||
/// Optional database store for DB-backed MCP config.
|
||||
store: Option<Arc<crate::history::Store>>,
|
||||
store: Option<Arc<dyn crate::db::Database>>,
|
||||
}
|
||||
|
||||
impl ExtensionManager {
|
||||
@@ -71,7 +71,7 @@ impl ExtensionManager {
|
||||
wasm_channels_dir: PathBuf,
|
||||
tunnel_url: Option<String>,
|
||||
user_id: String,
|
||||
store: Option<Arc<crate::history::Store>>,
|
||||
store: Option<Arc<dyn crate::db::Database>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
registry: ExtensionRegistry::new(),
|
||||
@@ -351,7 +351,7 @@ impl ExtensionManager {
|
||||
) -> Result<crate::tools::mcp::config::McpServersFile, crate::tools::mcp::config::ConfigError>
|
||||
{
|
||||
if let Some(ref store) = self.store {
|
||||
crate::tools::mcp::config::load_mcp_servers_from_db(store, &self.user_id).await
|
||||
crate::tools::mcp::config::load_mcp_servers_from_db(store.as_ref(), &self.user_id).await
|
||||
} else {
|
||||
crate::tools::mcp::config::load_mcp_servers().await
|
||||
}
|
||||
@@ -375,7 +375,8 @@ impl ExtensionManager {
|
||||
) -> Result<(), crate::tools::mcp::config::ConfigError> {
|
||||
config.validate()?;
|
||||
if let Some(ref store) = self.store {
|
||||
crate::tools::mcp::config::add_mcp_server_db(store, &self.user_id, config).await
|
||||
crate::tools::mcp::config::add_mcp_server_db(store.as_ref(), &self.user_id, config)
|
||||
.await
|
||||
} else {
|
||||
crate::tools::mcp::config::add_mcp_server(config).await
|
||||
}
|
||||
@@ -386,7 +387,8 @@ impl ExtensionManager {
|
||||
name: &str,
|
||||
) -> Result<(), crate::tools::mcp::config::ConfigError> {
|
||||
if let Some(ref store) = self.store {
|
||||
crate::tools::mcp::config::remove_mcp_server_db(store, &self.user_id, name).await
|
||||
crate::tools::mcp::config::remove_mcp_server_db(store.as_ref(), &self.user_id, name)
|
||||
.await
|
||||
} else {
|
||||
crate::tools::mcp::config::remove_mcp_server(name).await
|
||||
}
|
||||
@@ -461,7 +463,16 @@ impl ExtensionManager {
|
||||
name: &str,
|
||||
url: &str,
|
||||
) -> Result<InstallResult, ExtensionError> {
|
||||
// Download the WASM binary
|
||||
// Require HTTPS to prevent downgrade attacks
|
||||
if !url.starts_with("https://") {
|
||||
return Err(ExtensionError::InstallFailed(
|
||||
"Only HTTPS URLs are allowed for extension downloads".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// 50 MB cap to prevent disk-fill DoS
|
||||
const MAX_WASM_SIZE: usize = 50 * 1024 * 1024;
|
||||
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(60))
|
||||
.build()
|
||||
@@ -480,11 +491,36 @@ impl ExtensionManager {
|
||||
)));
|
||||
}
|
||||
|
||||
// Check Content-Length header before downloading the full body
|
||||
if let Some(len) = response.content_length() {
|
||||
if len as usize > MAX_WASM_SIZE {
|
||||
return Err(ExtensionError::InstallFailed(format!(
|
||||
"WASM binary too large ({} bytes, max {} bytes)",
|
||||
len, MAX_WASM_SIZE
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
let bytes = response
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?;
|
||||
|
||||
if bytes.len() > MAX_WASM_SIZE {
|
||||
return Err(ExtensionError::InstallFailed(format!(
|
||||
"WASM binary too large ({} bytes, max {} bytes)",
|
||||
bytes.len(),
|
||||
MAX_WASM_SIZE
|
||||
)));
|
||||
}
|
||||
|
||||
// Basic WASM magic number check (\0asm)
|
||||
if bytes.len() < 4 || &bytes[..4] != b"\0asm" {
|
||||
return Err(ExtensionError::InstallFailed(
|
||||
"Downloaded file is not a valid WASM binary (bad magic number)".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// Ensure tools directory exists
|
||||
tokio::fs::create_dir_all(&self.wasm_tools_dir)
|
||||
.await
|
||||
@@ -497,9 +533,10 @@ impl ExtensionManager {
|
||||
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
|
||||
|
||||
tracing::info!(
|
||||
"Installed WASM tool '{}' ({} bytes) to {}",
|
||||
"Installed WASM tool '{}' ({} bytes) from {} to {}",
|
||||
name,
|
||||
bytes.len(),
|
||||
url,
|
||||
wasm_path.display()
|
||||
);
|
||||
|
||||
|
||||
+5
-1
@@ -5,11 +5,15 @@
|
||||
//! - Learning from past executions
|
||||
//! - Analytics and metrics
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
mod analytics;
|
||||
mod store;
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
pub use analytics::{JobStats, ToolStats};
|
||||
#[cfg(feature = "postgres")]
|
||||
pub use store::Store;
|
||||
pub use store::{
|
||||
ConversationMessage, ConversationSummary, JobEventRecord, LlmCallRecord, SandboxJobRecord,
|
||||
SandboxJobSummary, Store,
|
||||
SandboxJobSummary, SettingRow,
|
||||
};
|
||||
|
||||
@@ -1,13 +1,18 @@
|
||||
//! PostgreSQL store for persisting agent data.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
#[cfg(feature = "postgres")]
|
||||
use deadpool_postgres::{Config, Pool, Runtime};
|
||||
use rust_decimal::Decimal;
|
||||
#[cfg(feature = "postgres")]
|
||||
use tokio_postgres::NoTls;
|
||||
use uuid::Uuid;
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
use crate::config::DatabaseConfig;
|
||||
#[cfg(feature = "postgres")]
|
||||
use crate::context::{ActionRecord, JobContext, JobState};
|
||||
#[cfg(feature = "postgres")]
|
||||
use crate::error::DatabaseError;
|
||||
|
||||
/// Record for an LLM call to be persisted.
|
||||
@@ -24,10 +29,12 @@ pub struct LlmCallRecord<'a> {
|
||||
}
|
||||
|
||||
/// Database store for the agent.
|
||||
#[cfg(feature = "postgres")]
|
||||
pub struct Store {
|
||||
pool: Pool,
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
impl Store {
|
||||
/// Create a new store and connect to the database.
|
||||
pub async fn new(config: &DatabaseConfig) -> Result<Self, DatabaseError> {
|
||||
@@ -144,7 +151,12 @@ impl Store {
|
||||
actual_cost, repair_attempts, created_at, started_at, completed_at
|
||||
) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17)
|
||||
ON CONFLICT (id) DO UPDATE SET
|
||||
title = EXCLUDED.title,
|
||||
description = EXCLUDED.description,
|
||||
category = EXCLUDED.category,
|
||||
status = EXCLUDED.status,
|
||||
estimated_cost = EXCLUDED.estimated_cost,
|
||||
estimated_time_secs = EXCLUDED.estimated_time_secs,
|
||||
actual_cost = EXCLUDED.actual_cost,
|
||||
repair_attempts = EXCLUDED.repair_attempts,
|
||||
started_at = EXCLUDED.started_at,
|
||||
@@ -220,6 +232,8 @@ impl Store {
|
||||
completed_at: row.get("completed_at"),
|
||||
transitions: Vec::new(), // Not loaded from DB for now
|
||||
metadata: serde_json::Value::Null,
|
||||
total_tokens_used: 0,
|
||||
max_tokens: 0,
|
||||
}))
|
||||
}
|
||||
None => Ok(None),
|
||||
@@ -464,6 +478,7 @@ pub struct SandboxJobSummary {
|
||||
pub interrupted: usize,
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
impl Store {
|
||||
/// Insert a new sandbox job into `agent_jobs`.
|
||||
pub async fn save_sandbox_job(&self, job: &SandboxJobRecord) -> Result<(), DatabaseError> {
|
||||
@@ -565,6 +580,90 @@ impl Store {
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// List sandbox jobs for a specific user, most recent first.
|
||||
pub async fn list_sandbox_jobs_for_user(
|
||||
&self,
|
||||
user_id: &str,
|
||||
) -> Result<Vec<SandboxJobRecord>, DatabaseError> {
|
||||
let conn = self.conn().await?;
|
||||
let rows = conn
|
||||
.query(
|
||||
r#"
|
||||
SELECT id, title, status, user_id, project_dir,
|
||||
success, failure_reason, created_at, started_at, completed_at
|
||||
FROM agent_jobs WHERE source = 'sandbox' AND user_id = $1
|
||||
ORDER BY created_at DESC
|
||||
"#,
|
||||
&[&user_id],
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(rows
|
||||
.iter()
|
||||
.map(|r| SandboxJobRecord {
|
||||
id: r.get("id"),
|
||||
task: r.get("title"),
|
||||
status: r.get("status"),
|
||||
user_id: r.get("user_id"),
|
||||
project_dir: r
|
||||
.get::<_, Option<String>>("project_dir")
|
||||
.unwrap_or_default(),
|
||||
success: r.get("success"),
|
||||
failure_reason: r.get("failure_reason"),
|
||||
created_at: r.get("created_at"),
|
||||
started_at: r.get("started_at"),
|
||||
completed_at: r.get("completed_at"),
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
/// Get a summary of sandbox job counts by status for a specific user.
|
||||
pub async fn sandbox_job_summary_for_user(
|
||||
&self,
|
||||
user_id: &str,
|
||||
) -> Result<SandboxJobSummary, DatabaseError> {
|
||||
let conn = self.conn().await?;
|
||||
let rows = conn
|
||||
.query(
|
||||
"SELECT status, COUNT(*) as cnt FROM agent_jobs WHERE source = 'sandbox' AND user_id = $1 GROUP BY status",
|
||||
&[&user_id],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut summary = SandboxJobSummary::default();
|
||||
for row in &rows {
|
||||
let status: String = row.get("status");
|
||||
let count: i64 = row.get("cnt");
|
||||
let c = count as usize;
|
||||
summary.total += c;
|
||||
match status.as_str() {
|
||||
"creating" => summary.creating += c,
|
||||
"running" => summary.running += c,
|
||||
"completed" => summary.completed += c,
|
||||
"failed" => summary.failed += c,
|
||||
"interrupted" => summary.interrupted += c,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
/// Check if a sandbox job belongs to a specific user.
|
||||
pub async fn sandbox_job_belongs_to_user(
|
||||
&self,
|
||||
job_id: Uuid,
|
||||
user_id: &str,
|
||||
) -> Result<bool, DatabaseError> {
|
||||
let conn = self.conn().await?;
|
||||
let row = conn
|
||||
.query_opt(
|
||||
"SELECT 1 FROM agent_jobs WHERE id = $1 AND user_id = $2 AND source = 'sandbox'",
|
||||
&[&job_id, &user_id],
|
||||
)
|
||||
.await?;
|
||||
Ok(row.is_some())
|
||||
}
|
||||
|
||||
/// Update sandbox job status and optional timestamps/result.
|
||||
pub async fn update_sandbox_job_status(
|
||||
&self,
|
||||
@@ -656,6 +755,7 @@ pub struct JobEventRecord {
|
||||
pub created_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
impl Store {
|
||||
/// Persist a job event (fire-and-forget from orchestrator handler).
|
||||
pub async fn save_job_event(
|
||||
@@ -728,10 +828,12 @@ impl Store {
|
||||
|
||||
// ==================== Routines ====================
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
use crate::agent::routine::{
|
||||
NotifyConfig, Routine, RoutineAction, RoutineGuardrails, RoutineRun, RunStatus, Trigger,
|
||||
};
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
impl Store {
|
||||
/// Create a new routine.
|
||||
pub async fn create_routine(&self, routine: &Routine) -> Result<(), DatabaseError> {
|
||||
@@ -1032,6 +1134,7 @@ impl Store {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
fn row_to_routine(row: &tokio_postgres::Row) -> Result<Routine, DatabaseError> {
|
||||
let trigger_type: String = row.get("trigger_type");
|
||||
let trigger_config: serde_json::Value = row.get("trigger_config");
|
||||
@@ -1076,6 +1179,7 @@ fn row_to_routine(row: &tokio_postgres::Row) -> Result<Routine, DatabaseError> {
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
fn row_to_routine_run(row: &tokio_postgres::Row) -> Result<RoutineRun, DatabaseError> {
|
||||
let status_str: String = row.get("status");
|
||||
let status: RunStatus = status_str
|
||||
@@ -1121,6 +1225,7 @@ pub struct ConversationMessage {
|
||||
pub created_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
impl Store {
|
||||
/// Ensure a conversation row exists for a given UUID.
|
||||
///
|
||||
@@ -1258,6 +1363,22 @@ impl Store {
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
/// Check whether a conversation belongs to the given user.
|
||||
pub async fn conversation_belongs_to_user(
|
||||
&self,
|
||||
conversation_id: Uuid,
|
||||
user_id: &str,
|
||||
) -> Result<bool, DatabaseError> {
|
||||
let conn = self.conn().await?;
|
||||
let row = conn
|
||||
.query_opt(
|
||||
"SELECT 1 FROM conversations WHERE id = $1 AND user_id = $2",
|
||||
&[&conversation_id, &user_id],
|
||||
)
|
||||
.await?;
|
||||
Ok(row.is_some())
|
||||
}
|
||||
|
||||
/// Load messages for a conversation with cursor-based pagination.
|
||||
///
|
||||
/// Returns `(messages_oldest_first, has_more)`.
|
||||
@@ -1375,6 +1496,7 @@ impl Store {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
fn parse_job_state(s: &str) -> JobState {
|
||||
match s {
|
||||
"pending" => JobState::Pending,
|
||||
@@ -1391,8 +1513,10 @@ fn parse_job_state(s: &str) -> JobState {
|
||||
|
||||
// ==================== Tool Failures ====================
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
use crate::agent::BrokenTool;
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
impl Store {
|
||||
/// Record a tool failure (upsert: increment count if exists).
|
||||
pub async fn record_tool_failure(
|
||||
@@ -1486,6 +1610,7 @@ pub struct SettingRow {
|
||||
pub updated_at: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
impl Store {
|
||||
/// Get a single setting by key.
|
||||
pub async fn get_setting(
|
||||
|
||||
@@ -44,6 +44,7 @@ pub mod channels;
|
||||
pub mod cli;
|
||||
pub mod config;
|
||||
pub mod context;
|
||||
pub mod db;
|
||||
pub mod error;
|
||||
pub mod estimation;
|
||||
pub mod evaluation;
|
||||
@@ -58,6 +59,7 @@ pub mod secrets;
|
||||
pub mod settings;
|
||||
pub mod setup;
|
||||
pub mod tools;
|
||||
pub mod util;
|
||||
pub mod worker;
|
||||
pub mod workspace;
|
||||
|
||||
|
||||
+4
-1
@@ -21,7 +21,10 @@ pub use provider::{
|
||||
ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ModelMetadata,
|
||||
Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, ToolDefinition, ToolResult,
|
||||
};
|
||||
pub use reasoning::{ActionPlan, Reasoning, ReasoningContext, RespondResult, ToolSelection};
|
||||
pub use reasoning::{
|
||||
ActionPlan, Reasoning, ReasoningContext, RespondOutput, RespondResult, TokenUsage,
|
||||
ToolSelection,
|
||||
};
|
||||
pub use rig_adapter::RigAdapter;
|
||||
pub use session::{SessionConfig, SessionManager, create_session_manager};
|
||||
|
||||
|
||||
@@ -222,6 +222,12 @@ impl LlmProvider for NearAiChatProvider {
|
||||
let messages: Vec<ChatCompletionMessage> =
|
||||
req.messages.into_iter().map(|m| m.into()).collect();
|
||||
|
||||
// NEAR AI cloud-api does not support multi-turn tool calling (rejects
|
||||
// any request containing role:"tool" messages with HTTP 400). Rewrite
|
||||
// tool-call / tool-result pairs into plain text so the conversation
|
||||
// history is preserved without using unsupported message roles.
|
||||
let messages = flatten_tool_messages(messages);
|
||||
|
||||
let tools: Vec<ChatCompletionTool> = req
|
||||
.tools
|
||||
.into_iter()
|
||||
@@ -367,6 +373,64 @@ struct ChatCompletionMessage {
|
||||
tool_calls: Option<Vec<ChatCompletionToolCall>>,
|
||||
}
|
||||
|
||||
/// Rewrite tool-call / tool-result messages into plain assistant/user text.
|
||||
///
|
||||
/// NEAR AI cloud-api does not support the OpenAI multi-turn tool-calling
|
||||
/// protocol (`role: "tool"` messages). This function converts:
|
||||
/// - Assistant messages with `tool_calls` → assistant text describing the calls
|
||||
/// - Tool result messages (`role: "tool"`) → user messages with the result
|
||||
///
|
||||
/// Non-tool messages pass through unchanged.
|
||||
fn flatten_tool_messages(messages: Vec<ChatCompletionMessage>) -> Vec<ChatCompletionMessage> {
|
||||
let has_tool_msgs = messages.iter().any(|m| m.role == "tool");
|
||||
if !has_tool_msgs {
|
||||
return messages;
|
||||
}
|
||||
|
||||
tracing::debug!("Flattening tool messages for NEAR AI compatibility");
|
||||
|
||||
messages
|
||||
.into_iter()
|
||||
.map(|msg| {
|
||||
if let (true, Some(calls)) = (msg.role == "assistant", &msg.tool_calls) {
|
||||
// Convert assistant tool_calls into descriptive text
|
||||
let mut parts: Vec<String> = Vec::new();
|
||||
if let Some(ref text) = msg.content {
|
||||
if !text.is_empty() {
|
||||
parts.push(text.clone());
|
||||
}
|
||||
}
|
||||
for tc in calls {
|
||||
parts.push(format!(
|
||||
"[Called tool `{}` with arguments: {}]",
|
||||
tc.function.name, tc.function.arguments
|
||||
));
|
||||
}
|
||||
ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: Some(parts.join("\n")),
|
||||
tool_call_id: None,
|
||||
name: None,
|
||||
tool_calls: None,
|
||||
}
|
||||
} else if msg.role == "tool" {
|
||||
// Convert tool result into a user message
|
||||
let tool_name = msg.name.as_deref().unwrap_or("unknown");
|
||||
let result = msg.content.as_deref().unwrap_or("");
|
||||
ChatCompletionMessage {
|
||||
role: "user".to_string(),
|
||||
content: Some(format!("[Tool `{}` returned: {}]", tool_name, result)),
|
||||
tool_call_id: None,
|
||||
name: None,
|
||||
tool_calls: None,
|
||||
}
|
||||
} else {
|
||||
msg
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
impl From<ChatMessage> for ChatCompletionMessage {
|
||||
fn from(msg: ChatMessage) -> Self {
|
||||
let role = match msg.role {
|
||||
@@ -544,4 +608,119 @@ mod tests {
|
||||
serde_json::from_str(&calls[0].function.arguments).expect("valid JSON string");
|
||||
assert_eq!(parsed["key"], "value");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_flatten_no_tool_messages_passthrough() {
|
||||
let messages = vec![
|
||||
ChatCompletionMessage {
|
||||
role: "system".to_string(),
|
||||
content: Some("You are helpful.".to_string()),
|
||||
tool_call_id: None,
|
||||
name: None,
|
||||
tool_calls: None,
|
||||
},
|
||||
ChatCompletionMessage {
|
||||
role: "user".to_string(),
|
||||
content: Some("Hello".to_string()),
|
||||
tool_call_id: None,
|
||||
name: None,
|
||||
tool_calls: None,
|
||||
},
|
||||
];
|
||||
let result = flatten_tool_messages(messages);
|
||||
assert_eq!(result.len(), 2);
|
||||
assert_eq!(result[0].role, "system");
|
||||
assert_eq!(result[1].role, "user");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_flatten_tool_call_and_result() {
|
||||
let messages = vec![
|
||||
ChatCompletionMessage {
|
||||
role: "user".to_string(),
|
||||
content: Some("test".to_string()),
|
||||
tool_call_id: None,
|
||||
name: None,
|
||||
tool_calls: None,
|
||||
},
|
||||
ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: None,
|
||||
tool_call_id: None,
|
||||
name: None,
|
||||
tool_calls: Some(vec![ChatCompletionToolCall {
|
||||
id: "call_1".to_string(),
|
||||
call_type: "function".to_string(),
|
||||
function: ChatCompletionToolCallFunction {
|
||||
name: "echo".to_string(),
|
||||
arguments: r#"{"message":"hi"}"#.to_string(),
|
||||
},
|
||||
}]),
|
||||
},
|
||||
ChatCompletionMessage {
|
||||
role: "tool".to_string(),
|
||||
content: Some("hi".to_string()),
|
||||
tool_call_id: Some("call_1".to_string()),
|
||||
name: Some("echo".to_string()),
|
||||
tool_calls: None,
|
||||
},
|
||||
];
|
||||
|
||||
let result = flatten_tool_messages(messages);
|
||||
assert_eq!(result.len(), 3);
|
||||
|
||||
// Assistant tool_calls → plain assistant text
|
||||
assert_eq!(result[1].role, "assistant");
|
||||
assert!(result[1].tool_calls.is_none());
|
||||
assert!(
|
||||
result[1]
|
||||
.content
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.contains("[Called tool `echo`")
|
||||
);
|
||||
|
||||
// Tool result → user message
|
||||
assert_eq!(result[2].role, "user");
|
||||
assert!(result[2].tool_call_id.is_none());
|
||||
assert!(
|
||||
result[2]
|
||||
.content
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.contains("[Tool `echo` returned: hi]")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_flatten_preserves_assistant_text_with_tool_calls() {
|
||||
let messages = vec![
|
||||
ChatCompletionMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: Some("Let me check that.".to_string()),
|
||||
tool_call_id: None,
|
||||
name: None,
|
||||
tool_calls: Some(vec![ChatCompletionToolCall {
|
||||
id: "call_1".to_string(),
|
||||
call_type: "function".to_string(),
|
||||
function: ChatCompletionToolCallFunction {
|
||||
name: "search".to_string(),
|
||||
arguments: r#"{"q":"test"}"#.to_string(),
|
||||
},
|
||||
}]),
|
||||
},
|
||||
ChatCompletionMessage {
|
||||
role: "tool".to_string(),
|
||||
content: Some("found it".to_string()),
|
||||
tool_call_id: Some("call_1".to_string()),
|
||||
name: Some("search".to_string()),
|
||||
tool_calls: None,
|
||||
},
|
||||
];
|
||||
|
||||
let result = flatten_tool_messages(messages);
|
||||
let text = result[0].content.as_ref().unwrap();
|
||||
assert!(text.starts_with("Let me check that."));
|
||||
assert!(text.contains("[Called tool `search`"));
|
||||
}
|
||||
}
|
||||
|
||||
+56
-17
@@ -115,6 +115,19 @@ pub struct ToolSelection {
|
||||
pub alternatives: Vec<String>,
|
||||
}
|
||||
|
||||
/// Token usage from a single LLM call.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct TokenUsage {
|
||||
pub input_tokens: u32,
|
||||
pub output_tokens: u32,
|
||||
}
|
||||
|
||||
impl TokenUsage {
|
||||
pub fn total(&self) -> u32 {
|
||||
self.input_tokens + self.output_tokens
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of a response with potential tool calls.
|
||||
///
|
||||
/// Used by the agent loop to handle tool execution before returning a final response.
|
||||
@@ -131,6 +144,13 @@ pub enum RespondResult {
|
||||
},
|
||||
}
|
||||
|
||||
/// A `RespondResult` bundled with the token usage from the LLM call that produced it.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RespondOutput {
|
||||
pub result: RespondResult,
|
||||
pub usage: TokenUsage,
|
||||
}
|
||||
|
||||
/// Reasoning engine for the agent.
|
||||
pub struct Reasoning {
|
||||
llm: Arc<dyn LlmProvider>,
|
||||
@@ -284,7 +304,8 @@ Respond in JSON format:
|
||||
/// tool calls as text for simple cases. Use `respond_with_tools()` when you
|
||||
/// need to actually execute tool calls in an agentic loop.
|
||||
pub async fn respond(&self, context: &ReasoningContext) -> Result<String, LlmError> {
|
||||
match self.respond_with_tools(context).await? {
|
||||
let output = self.respond_with_tools(context).await?;
|
||||
match output.result {
|
||||
RespondResult::Text(text) => Ok(text),
|
||||
RespondResult::ToolCalls {
|
||||
tool_calls: calls, ..
|
||||
@@ -299,15 +320,14 @@ Respond in JSON format:
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate a response that may include tool calls.
|
||||
/// Generate a response that may include tool calls, with token usage tracking.
|
||||
///
|
||||
/// Returns `RespondResult::ToolCalls` if the model wants to call tools,
|
||||
/// allowing the caller to execute them and continue the conversation.
|
||||
/// Returns `RespondResult::Text` when the model has a final text response.
|
||||
/// Returns `RespondOutput` containing the result and token usage from the LLM call.
|
||||
/// The caller should use `usage` to track cost/budget against the job.
|
||||
pub async fn respond_with_tools(
|
||||
&self,
|
||||
context: &ReasoningContext,
|
||||
) -> Result<RespondResult, LlmError> {
|
||||
) -> Result<RespondOutput, LlmError> {
|
||||
let system_prompt = self.build_conversation_prompt(context);
|
||||
|
||||
let mut messages = vec![ChatMessage::system(system_prompt)];
|
||||
@@ -322,12 +342,19 @@ Respond in JSON format:
|
||||
request.metadata = context.metadata.clone();
|
||||
|
||||
let response = self.llm.complete_with_tools(request).await?;
|
||||
let usage = TokenUsage {
|
||||
input_tokens: response.input_tokens,
|
||||
output_tokens: response.output_tokens,
|
||||
};
|
||||
|
||||
// If there were tool calls, return them for execution
|
||||
if !response.tool_calls.is_empty() {
|
||||
return Ok(RespondResult::ToolCalls {
|
||||
tool_calls: response.tool_calls,
|
||||
content: response.content,
|
||||
return Ok(RespondOutput {
|
||||
result: RespondResult::ToolCalls {
|
||||
tool_calls: response.tool_calls,
|
||||
content: response.content,
|
||||
},
|
||||
usage,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -341,17 +368,23 @@ Respond in JSON format:
|
||||
let recovered = recover_tool_calls_from_content(&content, &context.available_tools);
|
||||
if !recovered.is_empty() {
|
||||
let cleaned = clean_response(&content);
|
||||
return Ok(RespondResult::ToolCalls {
|
||||
tool_calls: recovered,
|
||||
content: if cleaned.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(cleaned)
|
||||
return Ok(RespondOutput {
|
||||
result: RespondResult::ToolCalls {
|
||||
tool_calls: recovered,
|
||||
content: if cleaned.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(cleaned)
|
||||
},
|
||||
},
|
||||
usage,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(RespondResult::Text(clean_response(&content)))
|
||||
Ok(RespondOutput {
|
||||
result: RespondResult::Text(clean_response(&content)),
|
||||
usage,
|
||||
})
|
||||
} else {
|
||||
// No tools, use simple completion
|
||||
let mut request = CompletionRequest::new(messages)
|
||||
@@ -360,7 +393,13 @@ Respond in JSON format:
|
||||
request.metadata = context.metadata.clone();
|
||||
|
||||
let response = self.llm.complete(request).await?;
|
||||
Ok(RespondResult::Text(clean_response(&response.content)))
|
||||
Ok(RespondOutput {
|
||||
result: RespondResult::Text(clean_response(&response.content)),
|
||||
usage: TokenUsage {
|
||||
input_tokens: response.input_tokens,
|
||||
output_tokens: response.output_tokens,
|
||||
},
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+21
-2
@@ -62,7 +62,7 @@ pub struct SessionManager {
|
||||
/// Prevents thundering herd during concurrent 401s.
|
||||
renewal_lock: Mutex<()>,
|
||||
/// Optional database store for persisting session to the settings table.
|
||||
store: RwLock<Option<Arc<crate::history::Store>>>,
|
||||
store: RwLock<Option<Arc<dyn crate::db::Database>>>,
|
||||
/// User ID for DB settings (default: "default").
|
||||
user_id: RwLock<String>,
|
||||
}
|
||||
@@ -125,7 +125,7 @@ impl SessionManager {
|
||||
/// When a store is attached, session tokens are saved to the `settings`
|
||||
/// table (key: `nearai.session_token`) in addition to the disk file.
|
||||
/// On load, DB is preferred over disk.
|
||||
pub async fn attach_store(&self, store: Arc<crate::history::Store>, user_id: &str) {
|
||||
pub async fn attach_store(&self, store: Arc<dyn crate::db::Database>, user_id: &str) {
|
||||
*self.store.write().await = Some(store);
|
||||
*self.user_id.write().await = user_id.to_string();
|
||||
|
||||
@@ -520,6 +520,25 @@ impl SessionManager {
|
||||
))
|
||||
})?;
|
||||
|
||||
// Restrictive permissions: session file contains a secret token
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let perms = std::fs::Permissions::from_mode(0o600);
|
||||
tokio::fs::set_permissions(&self.config.session_path, perms)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
LlmError::Io(std::io::Error::new(
|
||||
e.kind(),
|
||||
format!(
|
||||
"Failed to set permissions on {}: {}",
|
||||
self.config.session_path.display(),
|
||||
e
|
||||
),
|
||||
))
|
||||
})?;
|
||||
}
|
||||
|
||||
tracing::debug!("Session saved to {}", self.config.session_path.display());
|
||||
|
||||
// Also save to DB if a store is attached
|
||||
|
||||
+165
-52
@@ -17,13 +17,11 @@ use ironclaw::{
|
||||
web::log_layer::{LogBroadcaster, WebLogLayer},
|
||||
},
|
||||
cli::{
|
||||
Cli, Command, run_mcp_command, run_memory_command, run_pairing_command, run_status_command,
|
||||
run_tool_command,
|
||||
Cli, Command, run_mcp_command, run_pairing_command, run_status_command, run_tool_command,
|
||||
},
|
||||
config::Config,
|
||||
context::ContextManager,
|
||||
extensions::ExtensionManager,
|
||||
history::Store,
|
||||
llm::{SessionConfig, create_llm_provider, create_session_manager},
|
||||
orchestrator::{
|
||||
ContainerJobConfig, ContainerJobManager, OrchestratorApi, TokenStore,
|
||||
@@ -31,8 +29,7 @@ use ironclaw::{
|
||||
},
|
||||
pairing::PairingStore,
|
||||
safety::SafetyLayer,
|
||||
secrets::{PostgresSecretsStore, SecretsCrypto, SecretsStore},
|
||||
setup::{SetupConfig, SetupWizard},
|
||||
secrets::SecretsStore,
|
||||
tools::{
|
||||
ToolRegistry,
|
||||
mcp::{McpClient, McpSessionManager, config::load_mcp_servers_from_db, is_authenticated},
|
||||
@@ -41,6 +38,14 @@ use ironclaw::{
|
||||
workspace::{EmbeddingProvider, NearAiEmbeddings, OpenAiEmbeddings, Workspace},
|
||||
};
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
use ironclaw::secrets::LibSqlSecretsStore;
|
||||
#[cfg(feature = "postgres")]
|
||||
use ironclaw::secrets::PostgresSecretsStore;
|
||||
use ironclaw::secrets::SecretsCrypto;
|
||||
#[cfg(any(feature = "postgres", feature = "libsql"))]
|
||||
use ironclaw::setup::{SetupConfig, SetupWizard};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
let cli = Cli::parse();
|
||||
@@ -89,8 +94,6 @@ async fn main() -> anyhow::Result<()> {
|
||||
let config = Config::from_env()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||
let store = ironclaw::history::Store::new(&config.database).await?;
|
||||
store.run_migrations().await?;
|
||||
|
||||
// Set up embeddings if available
|
||||
let session = ironclaw::llm::create_session_manager(ironclaw::llm::SessionConfig {
|
||||
@@ -130,7 +133,14 @@ async fn main() -> anyhow::Result<()> {
|
||||
None
|
||||
};
|
||||
|
||||
return run_memory_command(mem_cmd.clone(), store.pool(), embeddings).await;
|
||||
// Create a Database-trait-backed workspace for the memory command
|
||||
let db: Arc<dyn ironclaw::db::Database> =
|
||||
ironclaw::db::connect_from_config(&config.database)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||
|
||||
return ironclaw::cli::run_memory_command_with_db(mem_cmd.clone(), db, embeddings)
|
||||
.await;
|
||||
}
|
||||
Some(Command::Pairing(pairing_cmd)) => {
|
||||
tracing_subscriber::fmt()
|
||||
@@ -216,6 +226,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
max_turns: *max_turns,
|
||||
model: model.clone(),
|
||||
timeout: std::time::Duration::from_secs(1800),
|
||||
allowed_tools: Vec::new(),
|
||||
};
|
||||
|
||||
let runtime = ironclaw::worker::ClaudeBridgeRuntime::new(config)
|
||||
@@ -235,12 +246,20 @@ async fn main() -> anyhow::Result<()> {
|
||||
// Load .env before running onboarding wizard
|
||||
let _ = dotenvy::dotenv();
|
||||
|
||||
let config = SetupConfig {
|
||||
skip_auth: *skip_auth,
|
||||
channels_only: *channels_only,
|
||||
};
|
||||
let mut wizard = SetupWizard::with_config(config);
|
||||
wizard.run().await?;
|
||||
#[cfg(any(feature = "postgres", feature = "libsql"))]
|
||||
{
|
||||
let config = SetupConfig {
|
||||
skip_auth: *skip_auth,
|
||||
channels_only: *channels_only,
|
||||
};
|
||||
let mut wizard = SetupWizard::with_config(config);
|
||||
wizard.run().await?;
|
||||
}
|
||||
#[cfg(not(any(feature = "postgres", feature = "libsql")))]
|
||||
{
|
||||
let _ = (skip_auth, channels_only);
|
||||
eprintln!("Onboarding wizard requires the 'postgres' or 'libsql' feature.");
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
None | Some(Command::Run) => {
|
||||
@@ -252,6 +271,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
let _ = dotenvy::dotenv();
|
||||
|
||||
// Enhanced first-run detection
|
||||
#[cfg(any(feature = "postgres", feature = "libsql"))]
|
||||
if !cli.no_onboard {
|
||||
if let Some(reason) = check_onboard_needed().await {
|
||||
println!("Onboarding needed: {}", reason);
|
||||
@@ -319,23 +339,86 @@ async fn main() -> anyhow::Result<()> {
|
||||
tracing::info!("Loaded configuration for agent: {}", config.agent.name);
|
||||
tracing::info!("LLM backend: {}", config.llm.backend);
|
||||
|
||||
// Initialize database store (optional for testing)
|
||||
let store = if cli.no_db {
|
||||
// Initialize database backend.
|
||||
//
|
||||
// Creates an `Arc<dyn Database>` that all consumers share.
|
||||
// Backend is selected by the `DATABASE_BACKEND` env var / config.
|
||||
//
|
||||
// NOTE: For simpler call sites (CLI commands, Memory handler) use the shared
|
||||
// helper `ironclaw::db::connect_from_config()`. This block is kept inline
|
||||
// because it also captures backend-specific handles (`pg_pool`, `libsql_db`)
|
||||
// needed by the secrets store.
|
||||
#[cfg(feature = "postgres")]
|
||||
let mut pg_pool: Option<deadpool_postgres::Pool> = None;
|
||||
#[cfg(feature = "libsql")]
|
||||
let mut libsql_db: Option<std::sync::Arc<libsql::Database>> = None;
|
||||
|
||||
let db: Option<Arc<dyn ironclaw::db::Database>> = if cli.no_db {
|
||||
tracing::warn!("Running without database connection");
|
||||
None
|
||||
} else {
|
||||
let store = Store::new(&config.database).await?;
|
||||
store.run_migrations().await?;
|
||||
tracing::info!("Database connected and migrations applied");
|
||||
match config.database.backend {
|
||||
#[cfg(feature = "libsql")]
|
||||
ironclaw::config::DatabaseBackend::LibSql => {
|
||||
use ironclaw::db::Database as _;
|
||||
use ironclaw::db::libsql_backend::LibSqlBackend;
|
||||
use secrecy::ExposeSecret as _;
|
||||
|
||||
let default_path = ironclaw::config::default_libsql_path();
|
||||
let db_path = config
|
||||
.database
|
||||
.libsql_path
|
||||
.as_deref()
|
||||
.unwrap_or(&default_path);
|
||||
|
||||
let backend = if let Some(ref url) = config.database.libsql_url {
|
||||
let token = config.database.libsql_auth_token.as_ref().ok_or_else(|| {
|
||||
anyhow::anyhow!("LIBSQL_AUTH_TOKEN is required when LIBSQL_URL is set")
|
||||
})?;
|
||||
LibSqlBackend::new_remote_replica(db_path, url, token.expose_secret()).await?
|
||||
} else {
|
||||
LibSqlBackend::new_local(db_path).await?
|
||||
};
|
||||
backend.run_migrations().await?;
|
||||
tracing::info!("libSQL database connected and migrations applied");
|
||||
|
||||
// Capture the Database handle for SecretsStore (connection-per-op)
|
||||
libsql_db = Some(backend.shared_db());
|
||||
|
||||
Some(Arc::new(backend) as Arc<dyn ironclaw::db::Database>)
|
||||
}
|
||||
#[cfg(feature = "postgres")]
|
||||
_ => {
|
||||
use ironclaw::db::Database as _;
|
||||
let pg = ironclaw::db::postgres::PgBackend::new(&config.database)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||
pg.run_migrations()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("{}", e))?;
|
||||
tracing::info!("PostgreSQL database connected and migrations applied");
|
||||
|
||||
pg_pool = Some(pg.pool());
|
||||
Some(Arc::new(pg) as Arc<dyn ironclaw::db::Database>)
|
||||
}
|
||||
#[cfg(not(feature = "postgres"))]
|
||||
_ => {
|
||||
anyhow::bail!(
|
||||
"No database backend available. Enable 'postgres' or 'libsql' feature."
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// Post-init operations using the database
|
||||
if let Some(ref db) = db {
|
||||
// One-time migration: move disk config files into the DB settings table.
|
||||
if let Err(e) = ironclaw::bootstrap::migrate_disk_to_db(&store, "default").await {
|
||||
if let Err(e) = ironclaw::bootstrap::migrate_disk_to_db(db.as_ref(), "default").await {
|
||||
tracing::warn!("Disk-to-DB settings migration failed: {}", e);
|
||||
}
|
||||
|
||||
// Reload config from DB now that we have a connection.
|
||||
// Priority: env var > DB setting > default.
|
||||
match Config::from_db(&store, "default", &bootstrap).await {
|
||||
match Config::from_db(db.as_ref(), "default", &bootstrap).await {
|
||||
Ok(db_config) => {
|
||||
config = db_config;
|
||||
tracing::info!("Configuration reloaded from database");
|
||||
@@ -348,19 +431,14 @@ async fn main() -> anyhow::Result<()> {
|
||||
}
|
||||
}
|
||||
|
||||
let store = Arc::new(store);
|
||||
|
||||
// Attach store to session manager so tokens save to DB too
|
||||
session.attach_store(Arc::clone(&store), "default").await;
|
||||
// Attach DB to session manager so tokens save to DB too
|
||||
session.attach_store(Arc::clone(db), "default").await;
|
||||
|
||||
// Mark any jobs left in "running" or "creating" state as "interrupted".
|
||||
if let Err(e) = store.cleanup_stale_sandbox_jobs().await {
|
||||
if let Err(e) = db.cleanup_stale_sandbox_jobs().await {
|
||||
tracing::warn!("Failed to cleanup stale sandbox jobs: {}", e);
|
||||
}
|
||||
|
||||
Some(store)
|
||||
};
|
||||
|
||||
}
|
||||
// Initialize LLM provider (clone session so we can reuse it for embeddings)
|
||||
let llm = create_llm_provider(&config.llm, session.clone())?;
|
||||
tracing::info!("LLM provider initialized: {}", llm.model_name());
|
||||
@@ -414,8 +492,8 @@ async fn main() -> anyhow::Result<()> {
|
||||
};
|
||||
|
||||
// Register memory tools if database is available
|
||||
if let Some(ref store) = store {
|
||||
let mut workspace = Workspace::new("default", store.pool());
|
||||
if let Some(ref db) = db {
|
||||
let mut workspace = Workspace::new_with_db("default", Arc::clone(db));
|
||||
if let Some(ref emb) = embeddings {
|
||||
workspace = workspace.with_embeddings(emb.clone());
|
||||
}
|
||||
@@ -438,20 +516,46 @@ async fn main() -> anyhow::Result<()> {
|
||||
tracing::info!("Builder mode enabled");
|
||||
}
|
||||
|
||||
// Create secrets store if master key is configured (needed for MCP auth and WASM channels)
|
||||
// Create secrets store if master key is configured (needed for MCP auth and WASM channels).
|
||||
//
|
||||
// When both `postgres` and `libsql` features are compiled, the runtime-selected
|
||||
// backend determines which store is created: whichever DB init branch ran will
|
||||
// have set its handle (pg_pool or libsql_db), and the or_else chain picks it up.
|
||||
let secrets_store: Option<Arc<dyn SecretsStore + Send + Sync>> =
|
||||
if let (Some(store), Some(master_key)) = (&store, config.secrets.master_key()) {
|
||||
if let Some(master_key) = config.secrets.master_key() {
|
||||
match SecretsCrypto::new(master_key.clone()) {
|
||||
Ok(crypto) => Some(Arc::new(PostgresSecretsStore::new(
|
||||
store.pool(),
|
||||
Arc::new(crypto),
|
||||
))),
|
||||
Ok(crypto) => {
|
||||
let crypto = Arc::new(crypto);
|
||||
let store: Option<Arc<dyn SecretsStore + Send + Sync>> = None;
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
let store = store.or_else(|| {
|
||||
libsql_db.take().map(|db| {
|
||||
Arc::new(LibSqlSecretsStore::new(db, Arc::clone(&crypto)))
|
||||
as Arc<dyn SecretsStore + Send + Sync>
|
||||
})
|
||||
});
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
let store = store.or_else(|| {
|
||||
pg_pool.as_ref().map(|pool| {
|
||||
Arc::new(PostgresSecretsStore::new(pool.clone(), Arc::clone(&crypto)))
|
||||
as Arc<dyn SecretsStore + Send + Sync>
|
||||
})
|
||||
});
|
||||
|
||||
store
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to initialize secrets crypto: {}", e);
|
||||
#[cfg(feature = "libsql")]
|
||||
let _ = libsql_db.take();
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
#[cfg(feature = "libsql")]
|
||||
let _ = libsql_db.take();
|
||||
None
|
||||
};
|
||||
|
||||
@@ -515,8 +619,8 @@ async fn main() -> anyhow::Result<()> {
|
||||
|
||||
let mcp_servers_future = async {
|
||||
if let Some(ref secrets) = secrets_store {
|
||||
let servers_result = if let Some(ref s) = store {
|
||||
load_mcp_servers_from_db(s, "default").await
|
||||
let servers_result = if let Some(ref d) = db {
|
||||
load_mcp_servers_from_db(d.as_ref(), "default").await
|
||||
} else {
|
||||
ironclaw::tools::mcp::config::load_mcp_servers().await
|
||||
};
|
||||
@@ -629,7 +733,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
config.channels.wasm_channels_dir.clone(),
|
||||
config.tunnel.public_url.clone(),
|
||||
"default".to_string(),
|
||||
store.clone(),
|
||||
db.clone(),
|
||||
));
|
||||
tools.register_extension_tools(Arc::clone(&manager));
|
||||
tracing::info!("Extension manager initialized with in-chat discovery tools");
|
||||
@@ -681,6 +785,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
claude_code_model: config.claude_code.model.clone(),
|
||||
claude_code_max_turns: config.claude_code.max_turns,
|
||||
claude_code_memory_limit_mb: config.claude_code.memory_limit_mb,
|
||||
claude_code_allowed_tools: config.claude_code.allowed_tools.clone(),
|
||||
};
|
||||
let jm = Arc::new(ContainerJobManager::new(job_config, token_store.clone()));
|
||||
|
||||
@@ -691,7 +796,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
token_store,
|
||||
job_event_tx: job_event_tx.clone(),
|
||||
prompt_queue: Arc::clone(&prompt_queue),
|
||||
store: store.clone(),
|
||||
store: db.clone(),
|
||||
};
|
||||
|
||||
tokio::spawn(async move {
|
||||
@@ -928,13 +1033,15 @@ async fn main() -> anyhow::Result<()> {
|
||||
};
|
||||
|
||||
// Create workspace for agent (shared with memory tools)
|
||||
let workspace = store.as_ref().map(|s| {
|
||||
let mut ws = Workspace::new("default", s.pool());
|
||||
let workspace = if let Some(ref db_ref) = db {
|
||||
let mut ws = Workspace::new_with_db("default", Arc::clone(db_ref));
|
||||
if let Some(ref emb) = embeddings {
|
||||
ws = ws.with_embeddings(emb.clone());
|
||||
}
|
||||
Arc::new(ws)
|
||||
});
|
||||
Some(Arc::new(ws))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Seed workspace with core identity files on first boot
|
||||
if let Some(ref ws) = workspace {
|
||||
@@ -972,7 +1079,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
tools.register_job_tools(
|
||||
Arc::clone(&context_manager),
|
||||
container_job_manager.clone(),
|
||||
store.clone(),
|
||||
db.clone(),
|
||||
);
|
||||
|
||||
// Add web gateway channel if configured
|
||||
@@ -987,8 +1094,8 @@ async fn main() -> anyhow::Result<()> {
|
||||
if let Some(ref ext_mgr) = extension_manager {
|
||||
gw = gw.with_extension_manager(Arc::clone(ext_mgr));
|
||||
}
|
||||
if let Some(ref s) = store {
|
||||
gw = gw.with_store(Arc::clone(s));
|
||||
if let Some(ref d) = db {
|
||||
gw = gw.with_store(Arc::clone(d));
|
||||
}
|
||||
if let Some(ref jm) = container_job_manager {
|
||||
gw = gw.with_job_manager(Arc::clone(jm));
|
||||
@@ -1025,7 +1132,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
|
||||
// Create and run the agent
|
||||
let deps = AgentDeps {
|
||||
store,
|
||||
store: db,
|
||||
llm,
|
||||
safety,
|
||||
tools,
|
||||
@@ -1059,11 +1166,17 @@ async fn main() -> anyhow::Result<()> {
|
||||
/// Check if onboarding is needed and return the reason.
|
||||
///
|
||||
/// Returns `Some(reason)` if onboarding should be triggered, `None` otherwise.
|
||||
#[cfg(any(feature = "postgres", feature = "libsql"))]
|
||||
async fn check_onboard_needed() -> Option<&'static str> {
|
||||
let bootstrap = ironclaw::bootstrap::BootstrapConfig::load();
|
||||
|
||||
// Database not configured (and not in env)
|
||||
if bootstrap.database_url.is_none() && std::env::var("DATABASE_URL").is_err() {
|
||||
let has_db = bootstrap.database_url.is_some()
|
||||
|| std::env::var("DATABASE_URL").is_ok()
|
||||
|| std::env::var("LIBSQL_PATH").is_ok()
|
||||
|| ironclaw::config::default_libsql_path().exists();
|
||||
|
||||
if !has_db {
|
||||
return Some("Database not configured");
|
||||
}
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ use tokio::sync::{Mutex, broadcast};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::channels::web::types::SseEvent;
|
||||
use crate::history::Store;
|
||||
use crate::db::Database;
|
||||
use crate::llm::{CompletionRequest, LlmProvider, ToolCompletionRequest};
|
||||
use crate::orchestrator::auth::{TokenStore, worker_auth_middleware};
|
||||
use crate::orchestrator::job_manager::ContainerJobManager;
|
||||
@@ -43,7 +43,7 @@ pub struct OrchestratorState {
|
||||
/// Buffered follow-up prompts for sandbox jobs, keyed by job_id.
|
||||
pub prompt_queue: Arc<Mutex<HashMap<Uuid, VecDeque<PendingPrompt>>>>,
|
||||
/// Database handle for persisting job events.
|
||||
pub store: Option<Arc<Store>>,
|
||||
pub store: Option<Arc<dyn Database>>,
|
||||
}
|
||||
|
||||
/// The orchestrator's internal API server.
|
||||
|
||||
@@ -14,6 +14,7 @@ use axum::http::StatusCode;
|
||||
use axum::middleware::Next;
|
||||
use axum::response::Response;
|
||||
use rand::Rng;
|
||||
use subtle::ConstantTimeEq;
|
||||
use tokio::sync::RwLock;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -38,13 +39,13 @@ impl TokenStore {
|
||||
token
|
||||
}
|
||||
|
||||
/// Validate a token for a specific job.
|
||||
/// Validate a token for a specific job (constant-time comparison).
|
||||
pub async fn validate(&self, job_id: Uuid, token: &str) -> bool {
|
||||
self.tokens
|
||||
.read()
|
||||
.await
|
||||
.get(&job_id)
|
||||
.map(|stored| stored == token)
|
||||
.map(|stored| stored.as_bytes().ct_eq(token.as_bytes()).into())
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
|
||||
@@ -58,6 +58,8 @@ pub struct ContainerJobConfig {
|
||||
pub claude_code_max_turns: u32,
|
||||
/// Memory limit in MB for Claude Code containers (heavier than workers).
|
||||
pub claude_code_memory_limit_mb: u64,
|
||||
/// Allowed tool patterns for Claude Code (passed as CLAUDE_CODE_ALLOWED_TOOLS env var).
|
||||
pub claude_code_allowed_tools: Vec<String>,
|
||||
}
|
||||
|
||||
impl Default for ContainerJobConfig {
|
||||
@@ -71,6 +73,7 @@ impl Default for ContainerJobConfig {
|
||||
claude_code_model: "sonnet".to_string(),
|
||||
claude_code_max_turns: 50,
|
||||
claude_code_memory_limit_mb: 4096,
|
||||
claude_code_allowed_tools: crate::config::ClaudeCodeConfig::default().allowed_tools,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -161,6 +164,29 @@ impl ContainerJobManager {
|
||||
};
|
||||
self.containers.write().await.insert(job_id, handle);
|
||||
|
||||
// Run the actual container creation. On any failure, revoke the token
|
||||
// and remove the handle so we don't leak resources.
|
||||
match self
|
||||
.create_job_inner(job_id, &token, project_dir, mode)
|
||||
.await
|
||||
{
|
||||
Ok(()) => Ok(token),
|
||||
Err(e) => {
|
||||
self.token_store.revoke(job_id).await;
|
||||
self.containers.write().await.remove(&job_id);
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Inner implementation of container creation (separated for cleanup).
|
||||
async fn create_job_inner(
|
||||
&self,
|
||||
job_id: Uuid,
|
||||
token: &str,
|
||||
project_dir: Option<PathBuf>,
|
||||
mode: JobMode,
|
||||
) -> Result<(), OrchestratorError> {
|
||||
// Connect to Docker
|
||||
let docker = connect_docker()
|
||||
.await
|
||||
@@ -219,11 +245,18 @@ impl ContainerJobManager {
|
||||
env_vec.push("IRONCLAW_WORKSPACE=/workspace".to_string());
|
||||
}
|
||||
|
||||
// Claude Code mode: mount host ~/.claude read-only for auth
|
||||
// Claude Code mode: mount host ~/.claude read-only for auth,
|
||||
// and pass the tool allowlist so the bridge can write settings.json.
|
||||
if mode == JobMode::ClaudeCode {
|
||||
if let Some(ref claude_dir) = self.config.claude_config_dir {
|
||||
binds.push(format!("{}:/home/sandbox/.claude:ro", claude_dir.display()));
|
||||
}
|
||||
if !self.config.claude_code_allowed_tools.is_empty() {
|
||||
env_vec.push(format!(
|
||||
"CLAUDE_CODE_ALLOWED_TOOLS={}",
|
||||
self.config.claude_code_allowed_tools.join(",")
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Memory limit: Claude Code gets more memory
|
||||
@@ -243,11 +276,7 @@ impl ContainerJobManager {
|
||||
network_mode: Some("bridge".to_string()),
|
||||
extra_hosts: Some(vec!["host.docker.internal:host-gateway".to_string()]),
|
||||
cap_drop: Some(vec!["ALL".to_string()]),
|
||||
cap_add: Some(vec![
|
||||
"CHOWN".to_string(),
|
||||
"SETUID".to_string(),
|
||||
"SETGID".to_string(),
|
||||
]),
|
||||
cap_add: Some(vec!["CHOWN".to_string()]),
|
||||
security_opt: Some(vec!["no-new-privileges:true".to_string()]),
|
||||
tmpfs: Some(
|
||||
[("/tmp".to_string(), "size=512M".to_string())]
|
||||
@@ -328,7 +357,7 @@ impl ContainerJobManager {
|
||||
"Created and started worker container"
|
||||
);
|
||||
|
||||
Ok(token)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stop a running container job.
|
||||
@@ -355,15 +384,18 @@ impl ContainerJobManager {
|
||||
})?;
|
||||
|
||||
// Stop the container (10 second grace period)
|
||||
let _ = docker
|
||||
if let Err(e) = docker
|
||||
.stop_container(
|
||||
&container_id,
|
||||
Some(bollard::container::StopContainerOptions { t: 10 }),
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
{
|
||||
tracing::warn!(job_id = %job_id, error = %e, "Failed to stop container (may already be stopped)");
|
||||
}
|
||||
|
||||
// Remove the container
|
||||
let _ = docker
|
||||
if let Err(e) = docker
|
||||
.remove_container(
|
||||
&container_id,
|
||||
Some(bollard::container::RemoveContainerOptions {
|
||||
@@ -371,7 +403,10 @@ impl ContainerJobManager {
|
||||
..Default::default()
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
{
|
||||
tracing::warn!(job_id = %job_id, error = %e, "Failed to remove container (may require manual cleanup)");
|
||||
}
|
||||
|
||||
// Update state
|
||||
if let Some(handle) = self.containers.write().await.get_mut(&job_id) {
|
||||
@@ -409,22 +444,33 @@ impl ContainerJobManager {
|
||||
};
|
||||
if let Some(cid) = container_id {
|
||||
if !cid.is_empty() {
|
||||
if let Ok(docker) = connect_docker().await {
|
||||
let _ = docker
|
||||
.stop_container(
|
||||
&cid,
|
||||
Some(bollard::container::StopContainerOptions { t: 5 }),
|
||||
)
|
||||
.await;
|
||||
let _ = docker
|
||||
.remove_container(
|
||||
&cid,
|
||||
Some(bollard::container::RemoveContainerOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
match connect_docker().await {
|
||||
Ok(docker) => {
|
||||
if let Err(e) = docker
|
||||
.stop_container(
|
||||
&cid,
|
||||
Some(bollard::container::StopContainerOptions { t: 5 }),
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(job_id = %job_id, error = %e, "Failed to stop completed container");
|
||||
}
|
||||
if let Err(e) = docker
|
||||
.remove_container(
|
||||
&cid,
|
||||
Some(bollard::container::RemoveContainerOptions {
|
||||
force: true,
|
||||
..Default::default()
|
||||
}),
|
||||
)
|
||||
.await
|
||||
{
|
||||
tracing::warn!(job_id = %job_id, error = %e, "Failed to remove completed container");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(job_id = %job_id, error = %e, "Failed to connect to Docker for container cleanup");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+11
-3
@@ -320,19 +320,27 @@ impl PairingStore {
|
||||
fn record_failed_approve(&self, channel: &str) -> Result<(), PairingStoreError> {
|
||||
let path = approve_attempts_path(&self.base_dir, channel)?;
|
||||
fs::create_dir_all(path.parent().unwrap())?;
|
||||
|
||||
// Open (or create) and lock before reading so concurrent callers
|
||||
// don't clobber each other's writes.
|
||||
let file = fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.truncate(false)
|
||||
.open(&path)?;
|
||||
file.lock_exclusive()?;
|
||||
let content = fs::read_to_string(&path).unwrap_or_default();
|
||||
let mut data: ApproveAttemptsFile = serde_json::from_str(&content).unwrap_or_default();
|
||||
|
||||
let mut data: ApproveAttemptsFile = fs::read_to_string(&path)
|
||||
.ok()
|
||||
.and_then(|c| serde_json::from_str(&c).ok())
|
||||
.unwrap_or_default();
|
||||
|
||||
let now = now_secs();
|
||||
data.failed_at.push(now);
|
||||
let cutoff = now.saturating_sub(PAIRING_APPROVE_RATE_WINDOW_SECS);
|
||||
data.failed_at.retain(|&t| t >= cutoff);
|
||||
|
||||
let json = serde_json::to_string_pretty(&data)?;
|
||||
fs::write(&path, json)?;
|
||||
fs4::FileExt::unlock(&file)?;
|
||||
|
||||
@@ -306,12 +306,11 @@ impl LeakDetector {
|
||||
})?;
|
||||
}
|
||||
|
||||
// Scan body if present and valid UTF-8
|
||||
// Scan body if present. Use lossy UTF-8 conversion so a leading
|
||||
// non-UTF8 byte can't be used to skip scanning entirely.
|
||||
if let Some(body_bytes) = body {
|
||||
if let Ok(body_str) = std::str::from_utf8(body_bytes) {
|
||||
self.scan_and_clean(body_str)?;
|
||||
}
|
||||
// Binary bodies are not scanned (could add hex pattern detection later)
|
||||
let body_str = String::from_utf8_lossy(body_bytes);
|
||||
self.scan_and_clean(&body_str)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -705,4 +704,17 @@ mod tests {
|
||||
let result = detector.scan_http_request("https://api.example.com/webhook", &[], Some(body));
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_scan_http_request_blocks_secret_in_binary_body() {
|
||||
let detector = LeakDetector::new();
|
||||
|
||||
// Attacker prepends a non-UTF8 byte to bypass strict from_utf8 check.
|
||||
// The lossy conversion should still detect the secret.
|
||||
let mut body = vec![0xFF]; // invalid UTF-8 leading byte
|
||||
body.extend_from_slice(b"sk-proj-test1234567890abcdefghij");
|
||||
|
||||
let result = detector.scan_http_request("https://api.example.com/exfil", &[], Some(&body));
|
||||
assert!(result.is_err(), "binary body should still be scanned");
|
||||
}
|
||||
}
|
||||
|
||||
+21
-5
@@ -98,15 +98,15 @@ impl SafetyLayer {
|
||||
was_modified: true,
|
||||
};
|
||||
}
|
||||
if violations
|
||||
let force_sanitize = violations
|
||||
.iter()
|
||||
.any(|rule| rule.action == crate::safety::PolicyAction::Sanitize)
|
||||
{
|
||||
.any(|rule| rule.action == crate::safety::PolicyAction::Sanitize);
|
||||
if force_sanitize {
|
||||
was_modified = true;
|
||||
}
|
||||
|
||||
// Run sanitization if enabled
|
||||
if self.config.injection_check_enabled {
|
||||
// Run sanitization once: if injection_check is enabled OR policy requires it
|
||||
if self.config.injection_check_enabled || force_sanitize {
|
||||
let mut sanitized = self.sanitizer.sanitize(&content);
|
||||
sanitized.was_modified = sanitized.was_modified || was_modified;
|
||||
sanitized
|
||||
@@ -190,4 +190,20 @@ mod tests {
|
||||
assert!(wrapped.contains("sanitized=\"true\""));
|
||||
assert!(wrapped.contains("Hello <world>"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sanitize_action_forces_sanitization_when_injection_check_disabled() {
|
||||
let config = SafetyConfig {
|
||||
max_output_length: 100_000,
|
||||
injection_check_enabled: false,
|
||||
};
|
||||
let safety = SafetyLayer::new(&config);
|
||||
|
||||
// Content with an injection-like pattern that a policy might flag
|
||||
let output = safety.sanitize_tool_output("test", "normal text");
|
||||
// With injection_check disabled and no policy violations, content
|
||||
// should pass through unmodified
|
||||
assert_eq!(output.content, "normal text");
|
||||
assert!(!output.was_modified);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -279,11 +279,7 @@ impl ContainerRunner {
|
||||
network_mode: Some("bridge".to_string()),
|
||||
// Security: drop all capabilities and add back only what's needed
|
||||
cap_drop: Some(vec!["ALL".to_string()]),
|
||||
cap_add: Some(vec![
|
||||
"CHOWN".to_string(),
|
||||
"SETUID".to_string(),
|
||||
"SETGID".to_string(),
|
||||
]),
|
||||
cap_add: Some(vec!["CHOWN".to_string()]),
|
||||
// Prevent privilege escalation
|
||||
security_opt: Some(vec!["no-new-privileges:true".to_string()]),
|
||||
// Read-only root filesystem (workspace is still writable if policy allows)
|
||||
|
||||
+5
-1
@@ -64,7 +64,11 @@ mod store;
|
||||
mod types;
|
||||
|
||||
pub use crypto::SecretsCrypto;
|
||||
pub use store::{PostgresSecretsStore, SecretsStore};
|
||||
#[cfg(feature = "libsql")]
|
||||
pub use store::LibSqlSecretsStore;
|
||||
#[cfg(feature = "postgres")]
|
||||
pub use store::PostgresSecretsStore;
|
||||
pub use store::SecretsStore;
|
||||
pub use types::{
|
||||
CreateSecretParams, CredentialLocation, CredentialMapping, DecryptedSecret, Secret,
|
||||
SecretError, SecretRef,
|
||||
|
||||
@@ -10,6 +10,7 @@ use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::Utc;
|
||||
#[cfg(feature = "postgres")]
|
||||
use deadpool_postgres::Pool;
|
||||
use secrecy::ExposeSecret;
|
||||
use uuid::Uuid;
|
||||
@@ -61,11 +62,13 @@ pub trait SecretsStore: Send + Sync {
|
||||
}
|
||||
|
||||
/// PostgreSQL implementation of SecretsStore.
|
||||
#[cfg(feature = "postgres")]
|
||||
pub struct PostgresSecretsStore {
|
||||
pool: Pool,
|
||||
crypto: Arc<SecretsCrypto>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
impl PostgresSecretsStore {
|
||||
/// Create a new store with the given database pool and crypto instance.
|
||||
pub fn new(pool: Pool, crypto: Arc<SecretsCrypto>) -> Self {
|
||||
@@ -73,6 +76,7 @@ impl PostgresSecretsStore {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
#[async_trait]
|
||||
impl SecretsStore for PostgresSecretsStore {
|
||||
async fn create(
|
||||
@@ -283,6 +287,7 @@ impl SecretsStore for PostgresSecretsStore {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
fn row_to_secret(row: &tokio_postgres::Row) -> Secret {
|
||||
Secret {
|
||||
id: row.get("id"),
|
||||
@@ -299,6 +304,332 @@ fn row_to_secret(row: &tokio_postgres::Row) -> Secret {
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== libSQL implementation ====================
|
||||
|
||||
/// libSQL/Turso implementation of SecretsStore.
|
||||
///
|
||||
/// Holds an `Arc<Database>` handle and creates a fresh connection per operation,
|
||||
/// matching the connection-per-request pattern used by the main `LibSqlBackend`.
|
||||
#[cfg(feature = "libsql")]
|
||||
pub struct LibSqlSecretsStore {
|
||||
db: Arc<libsql::Database>,
|
||||
crypto: Arc<SecretsCrypto>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
impl LibSqlSecretsStore {
|
||||
/// Create a new store with the given shared libsql database handle and crypto instance.
|
||||
pub fn new(db: Arc<libsql::Database>, crypto: Arc<SecretsCrypto>) -> Self {
|
||||
Self { db, crypto }
|
||||
}
|
||||
|
||||
fn connect(&self) -> Result<libsql::Connection, SecretError> {
|
||||
self.db
|
||||
.connect()
|
||||
.map_err(|e| SecretError::Database(format!("Connection failed: {}", e)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
#[async_trait]
|
||||
impl SecretsStore for LibSqlSecretsStore {
|
||||
async fn create(
|
||||
&self,
|
||||
user_id: &str,
|
||||
params: CreateSecretParams,
|
||||
) -> Result<Secret, SecretError> {
|
||||
let plaintext = params.value.expose_secret().as_bytes();
|
||||
let (encrypted_value, key_salt) = self.crypto.encrypt(plaintext)?;
|
||||
|
||||
let id = Uuid::new_v4();
|
||||
let now = Utc::now();
|
||||
let now_str = now.to_rfc3339_opts(chrono::SecondsFormat::Millis, true);
|
||||
let expires_at_str = params
|
||||
.expires_at
|
||||
.map(|dt| dt.to_rfc3339_opts(chrono::SecondsFormat::Millis, true));
|
||||
|
||||
// Start transaction for atomic upsert + read-back
|
||||
let conn = self.connect()?;
|
||||
let tx = conn
|
||||
.transaction()
|
||||
.await
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
|
||||
tx.execute(
|
||||
r#"
|
||||
INSERT INTO secrets (id, user_id, name, encrypted_value, key_salt, provider, expires_at, created_at, updated_at)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?8)
|
||||
ON CONFLICT (user_id, name) DO UPDATE SET
|
||||
encrypted_value = excluded.encrypted_value,
|
||||
key_salt = excluded.key_salt,
|
||||
provider = excluded.provider,
|
||||
expires_at = excluded.expires_at,
|
||||
updated_at = ?8
|
||||
"#,
|
||||
libsql::params![
|
||||
id.to_string(),
|
||||
user_id,
|
||||
params.name.as_str(),
|
||||
libsql::Value::Blob(encrypted_value.clone()),
|
||||
libsql::Value::Blob(key_salt.clone()),
|
||||
libsql_opt_text(params.provider.as_deref()),
|
||||
libsql_opt_text(expires_at_str.as_deref()),
|
||||
now_str.as_str(),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
|
||||
// Read back the row (may have been upserted)
|
||||
let mut rows = tx
|
||||
.query(
|
||||
r#"
|
||||
SELECT id, user_id, name, encrypted_value, key_salt, provider, expires_at,
|
||||
last_used_at, usage_count, created_at, updated_at
|
||||
FROM secrets
|
||||
WHERE user_id = ?1 AND name = ?2
|
||||
"#,
|
||||
libsql::params![user_id, params.name.as_str()],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
|
||||
let row = rows
|
||||
.next()
|
||||
.await
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?
|
||||
.ok_or_else(|| SecretError::Database("Insert succeeded but row not found".into()))?;
|
||||
|
||||
let secret = libsql_row_to_secret(&row)?;
|
||||
|
||||
tx.commit()
|
||||
.await
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
|
||||
Ok(secret)
|
||||
}
|
||||
|
||||
async fn get(&self, user_id: &str, name: &str) -> Result<Secret, SecretError> {
|
||||
let conn = self.connect()?;
|
||||
let mut rows = conn
|
||||
.query(
|
||||
r#"
|
||||
SELECT id, user_id, name, encrypted_value, key_salt, provider, expires_at,
|
||||
last_used_at, usage_count, created_at, updated_at
|
||||
FROM secrets
|
||||
WHERE user_id = ?1 AND name = ?2
|
||||
"#,
|
||||
libsql::params![user_id, name],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
|
||||
match rows
|
||||
.next()
|
||||
.await
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?
|
||||
{
|
||||
Some(row) => {
|
||||
let secret = libsql_row_to_secret(&row)?;
|
||||
|
||||
if let Some(expires_at) = secret.expires_at {
|
||||
if expires_at < Utc::now() {
|
||||
return Err(SecretError::Expired);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(secret)
|
||||
}
|
||||
None => Err(SecretError::NotFound(name.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_decrypted(
|
||||
&self,
|
||||
user_id: &str,
|
||||
name: &str,
|
||||
) -> Result<DecryptedSecret, SecretError> {
|
||||
let secret = self.get(user_id, name).await?;
|
||||
self.crypto
|
||||
.decrypt(&secret.encrypted_value, &secret.key_salt)
|
||||
}
|
||||
|
||||
async fn exists(&self, user_id: &str, name: &str) -> Result<bool, SecretError> {
|
||||
let conn = self.connect()?;
|
||||
let mut rows = conn
|
||||
.query(
|
||||
"SELECT 1 FROM secrets WHERE user_id = ?1 AND name = ?2",
|
||||
libsql::params![user_id, name],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
|
||||
Ok(rows
|
||||
.next()
|
||||
.await
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?
|
||||
.is_some())
|
||||
}
|
||||
|
||||
async fn list(&self, user_id: &str) -> Result<Vec<SecretRef>, SecretError> {
|
||||
let conn = self.connect()?;
|
||||
let mut rows = conn
|
||||
.query(
|
||||
"SELECT name, provider FROM secrets WHERE user_id = ?1 ORDER BY name",
|
||||
libsql::params![user_id],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
|
||||
let mut refs = Vec::new();
|
||||
while let Some(row) = rows
|
||||
.next()
|
||||
.await
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?
|
||||
{
|
||||
refs.push(SecretRef {
|
||||
name: row.get::<String>(0).unwrap_or_default(),
|
||||
provider: row.get::<String>(1).ok(),
|
||||
});
|
||||
}
|
||||
Ok(refs)
|
||||
}
|
||||
|
||||
async fn delete(&self, user_id: &str, name: &str) -> Result<bool, SecretError> {
|
||||
let conn = self.connect()?;
|
||||
let affected = conn
|
||||
.execute(
|
||||
"DELETE FROM secrets WHERE user_id = ?1 AND name = ?2",
|
||||
libsql::params![user_id, name],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
|
||||
Ok(affected > 0)
|
||||
}
|
||||
|
||||
async fn record_usage(&self, secret_id: Uuid) -> Result<(), SecretError> {
|
||||
let now = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true);
|
||||
let conn = self.connect()?;
|
||||
|
||||
conn.execute(
|
||||
r#"
|
||||
UPDATE secrets
|
||||
SET last_used_at = ?1, usage_count = usage_count + 1
|
||||
WHERE id = ?2
|
||||
"#,
|
||||
libsql::params![now.as_str(), secret_id.to_string()],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn is_accessible(
|
||||
&self,
|
||||
user_id: &str,
|
||||
secret_name: &str,
|
||||
allowed_secrets: &[String],
|
||||
) -> Result<bool, SecretError> {
|
||||
if !self.exists(user_id, secret_name).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
for pattern in allowed_secrets {
|
||||
if pattern == secret_name {
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
if let Some(prefix) = pattern.strip_suffix('*') {
|
||||
if secret_name.starts_with(prefix) {
|
||||
return Ok(true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
fn libsql_opt_text(s: Option<&str>) -> libsql::Value {
|
||||
match s {
|
||||
Some(s) => libsql::Value::Text(s.to_string()),
|
||||
None => libsql::Value::Null,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
fn libsql_parse_timestamp(s: &str) -> Result<chrono::DateTime<Utc>, SecretError> {
|
||||
if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(s) {
|
||||
return Ok(dt.with_timezone(&Utc));
|
||||
}
|
||||
if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f") {
|
||||
return Ok(ndt.and_utc());
|
||||
}
|
||||
if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S") {
|
||||
return Ok(ndt.and_utc());
|
||||
}
|
||||
Err(SecretError::Database(format!(
|
||||
"unparseable timestamp: {:?}",
|
||||
s
|
||||
)))
|
||||
}
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
fn libsql_row_to_secret(row: &libsql::Row) -> Result<Secret, SecretError> {
|
||||
let id_str: String = row
|
||||
.get(0)
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
let user_id: String = row
|
||||
.get(1)
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
let name: String = row
|
||||
.get(2)
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
let encrypted_value: Vec<u8> = row
|
||||
.get(3)
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
let key_salt: Vec<u8> = row
|
||||
.get(4)
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
let provider: Option<String> = row.get::<String>(5).ok().filter(|s| !s.is_empty());
|
||||
let expires_at = row
|
||||
.get::<String>(6)
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty())
|
||||
.and_then(|s| libsql_parse_timestamp(&s).ok());
|
||||
let last_used_at = row
|
||||
.get::<String>(7)
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty())
|
||||
.and_then(|s| libsql_parse_timestamp(&s).ok());
|
||||
let usage_count: i64 = row.get::<i64>(8).unwrap_or(0);
|
||||
let created_at_str: String = row
|
||||
.get(9)
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
let updated_at_str: String = row
|
||||
.get(10)
|
||||
.map_err(|e| SecretError::Database(e.to_string()))?;
|
||||
|
||||
Ok(Secret {
|
||||
id: id_str
|
||||
.parse()
|
||||
.map_err(|e: uuid::Error| SecretError::Database(e.to_string()))?,
|
||||
user_id,
|
||||
name,
|
||||
encrypted_value,
|
||||
key_salt,
|
||||
provider,
|
||||
expires_at,
|
||||
last_used_at,
|
||||
usage_count,
|
||||
created_at: libsql_parse_timestamp(&created_at_str)?,
|
||||
updated_at: libsql_parse_timestamp(&updated_at_str)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// In-memory implementation for testing.
|
||||
#[cfg(test)]
|
||||
pub mod testing {
|
||||
|
||||
@@ -15,6 +15,10 @@ pub struct Settings {
|
||||
pub onboard_completed: bool,
|
||||
|
||||
// === Step 1: Database ===
|
||||
/// Database backend: "postgres" or "libsql".
|
||||
#[serde(default)]
|
||||
pub database_backend: Option<String>,
|
||||
|
||||
/// Database connection URL (postgres://...).
|
||||
#[serde(default)]
|
||||
pub database_url: Option<String>,
|
||||
@@ -23,6 +27,14 @@ pub struct Settings {
|
||||
#[serde(default)]
|
||||
pub database_pool_size: Option<usize>,
|
||||
|
||||
/// Path to local libSQL database file.
|
||||
#[serde(default)]
|
||||
pub libsql_path: Option<String>,
|
||||
|
||||
/// Turso cloud URL for remote replica sync.
|
||||
#[serde(default)]
|
||||
pub libsql_url: Option<String>,
|
||||
|
||||
// === Step 2: Security ===
|
||||
/// Source for the secrets master key.
|
||||
#[serde(default)]
|
||||
|
||||
+15
-4
@@ -12,7 +12,9 @@ use reqwest::Client;
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::secrets::{CreateSecretParams, PostgresSecretsStore, SecretsCrypto, SecretsStore};
|
||||
#[cfg(feature = "postgres")]
|
||||
use crate::secrets::SecretsCrypto;
|
||||
use crate::secrets::{CreateSecretParams, SecretsStore};
|
||||
use crate::settings::Settings;
|
||||
use crate::setup::prompts::{
|
||||
confirm, input, optional_input, print_error, print_info, print_success, secret_input,
|
||||
@@ -20,15 +22,24 @@ use crate::setup::prompts::{
|
||||
|
||||
/// Context for saving secrets during setup.
|
||||
pub struct SecretsContext {
|
||||
store: PostgresSecretsStore,
|
||||
store: Arc<dyn SecretsStore>,
|
||||
user_id: String,
|
||||
}
|
||||
|
||||
impl SecretsContext {
|
||||
/// Create a new secrets context.
|
||||
/// Create a new secrets context from a trait-object store.
|
||||
pub fn from_store(store: Arc<dyn SecretsStore>, user_id: &str) -> Self {
|
||||
Self {
|
||||
store,
|
||||
user_id: user_id.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new secrets context from a PostgreSQL pool and crypto.
|
||||
#[cfg(feature = "postgres")]
|
||||
pub fn new(pool: deadpool_postgres::Pool, crypto: Arc<SecretsCrypto>, user_id: &str) -> Self {
|
||||
Self {
|
||||
store: PostgresSecretsStore::new(pool, crypto),
|
||||
store: Arc::new(crate::secrets::PostgresSecretsStore::new(pool, crypto)),
|
||||
user_id: user_id.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
|
||||
mod channels;
|
||||
mod prompts;
|
||||
#[cfg(any(feature = "postgres", feature = "libsql"))]
|
||||
mod wizard;
|
||||
|
||||
pub use channels::{
|
||||
@@ -29,4 +30,5 @@ pub use prompts::{
|
||||
confirm, input, optional_input, print_error, print_header, print_info, print_step,
|
||||
print_success, secret_input, select_many, select_one,
|
||||
};
|
||||
#[cfg(any(feature = "postgres", feature = "libsql"))]
|
||||
pub use wizard::{SetupConfig, SetupWizard};
|
||||
|
||||
+306
-37
@@ -12,15 +12,17 @@
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
use deadpool_postgres::{Config as PoolConfig, Runtime};
|
||||
use secrecy::SecretString;
|
||||
#[cfg(feature = "postgres")]
|
||||
use tokio_postgres::NoTls;
|
||||
|
||||
use crate::channels::wasm::{
|
||||
ChannelCapabilitiesFile, available_channel_names, install_bundled_channel,
|
||||
};
|
||||
use crate::llm::{SessionConfig, SessionManager};
|
||||
use crate::secrets::SecretsCrypto;
|
||||
use crate::secrets::{SecretsCrypto, SecretsStore};
|
||||
use crate::settings::{KeySource, Settings};
|
||||
use crate::setup::channels::{
|
||||
SecretsContext, setup_http, setup_telegram, setup_tunnel, setup_wasm_channel,
|
||||
@@ -66,8 +68,12 @@ pub struct SetupWizard {
|
||||
config: SetupConfig,
|
||||
settings: Settings,
|
||||
session_manager: Option<Arc<SessionManager>>,
|
||||
/// Database pool (created during setup).
|
||||
/// Database pool (created during setup, postgres only).
|
||||
#[cfg(feature = "postgres")]
|
||||
db_pool: Option<deadpool_postgres::Pool>,
|
||||
/// libSQL backend (created during setup, libsql only).
|
||||
#[cfg(feature = "libsql")]
|
||||
db_backend: Option<crate::db::libsql_backend::LibSqlBackend>,
|
||||
/// Secrets crypto (created during setup).
|
||||
secrets_crypto: Option<Arc<SecretsCrypto>>,
|
||||
}
|
||||
@@ -79,7 +85,10 @@ impl SetupWizard {
|
||||
config: SetupConfig::default(),
|
||||
settings: Settings::load(),
|
||||
session_manager: None,
|
||||
#[cfg(feature = "postgres")]
|
||||
db_pool: None,
|
||||
#[cfg(feature = "libsql")]
|
||||
db_backend: None,
|
||||
secrets_crypto: None,
|
||||
}
|
||||
}
|
||||
@@ -90,7 +99,10 @@ impl SetupWizard {
|
||||
config,
|
||||
settings: Settings::load(),
|
||||
session_manager: None,
|
||||
#[cfg(feature = "postgres")]
|
||||
db_pool: None,
|
||||
#[cfg(feature = "libsql")]
|
||||
db_backend: None,
|
||||
secrets_crypto: None,
|
||||
}
|
||||
}
|
||||
@@ -153,19 +165,48 @@ impl SetupWizard {
|
||||
|
||||
/// Step 1: Database connection.
|
||||
async fn step_database(&mut self) -> Result<(), SetupError> {
|
||||
// Check if we have an existing URL in env or settings
|
||||
// Determine which backend to use based on compile-time features.
|
||||
// When both features are enabled, prefer the currently configured backend
|
||||
// or default to postgres.
|
||||
#[cfg(all(feature = "postgres", feature = "libsql"))]
|
||||
{
|
||||
let backend = std::env::var("DATABASE_BACKEND")
|
||||
.ok()
|
||||
.or_else(|| self.settings.database_backend.clone())
|
||||
.unwrap_or_else(|| "postgres".to_string());
|
||||
|
||||
if backend == "libsql" || backend == "turso" || backend == "sqlite" {
|
||||
return self.step_database_libsql().await;
|
||||
}
|
||||
return self.step_database_postgres().await;
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "postgres", not(feature = "libsql")))]
|
||||
{
|
||||
return self.step_database_postgres().await;
|
||||
}
|
||||
|
||||
#[cfg(all(feature = "libsql", not(feature = "postgres")))]
|
||||
{
|
||||
return self.step_database_libsql().await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Step 1 (postgres): Database connection via PostgreSQL URL.
|
||||
#[cfg(feature = "postgres")]
|
||||
async fn step_database_postgres(&mut self) -> Result<(), SetupError> {
|
||||
self.settings.database_backend = Some("postgres".to_string());
|
||||
|
||||
let existing_url = std::env::var("DATABASE_URL")
|
||||
.ok()
|
||||
.or_else(|| self.settings.database_url.clone());
|
||||
|
||||
if let Some(ref url) = existing_url {
|
||||
// Mask the password for display
|
||||
let display_url = mask_password_in_url(url);
|
||||
print_info(&format!("Existing database URL: {}", display_url));
|
||||
|
||||
if confirm("Use this database?", true).map_err(SetupError::Io)? {
|
||||
// Test the connection
|
||||
if let Err(e) = self.test_database_connection(url).await {
|
||||
if let Err(e) = self.test_database_connection_postgres(url).await {
|
||||
print_error(&format!("Connection failed: {}", e));
|
||||
print_info("Let's configure a new database URL.");
|
||||
} else {
|
||||
@@ -176,7 +217,6 @@ impl SetupWizard {
|
||||
}
|
||||
}
|
||||
|
||||
// Prompt for new URL
|
||||
println!();
|
||||
print_info("Enter your PostgreSQL connection URL.");
|
||||
print_info("Format: postgres://user:password@host:port/database");
|
||||
@@ -190,15 +230,13 @@ impl SetupWizard {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Test the connection
|
||||
print_info("Testing connection...");
|
||||
match self.test_database_connection(&url).await {
|
||||
match self.test_database_connection_postgres(&url).await {
|
||||
Ok(()) => {
|
||||
print_success("Database connection successful");
|
||||
|
||||
// Ask if we should run migrations
|
||||
if confirm("Run database migrations?", true).map_err(SetupError::Io)? {
|
||||
self.run_migrations().await?;
|
||||
self.run_migrations_postgres().await?;
|
||||
}
|
||||
|
||||
self.settings.database_url = Some(url);
|
||||
@@ -216,8 +254,115 @@ impl SetupWizard {
|
||||
}
|
||||
}
|
||||
|
||||
/// Test database connection and store the pool.
|
||||
async fn test_database_connection(&mut self, url: &str) -> Result<(), SetupError> {
|
||||
/// Step 1 (libsql): Database connection via local file or Turso remote replica.
|
||||
#[cfg(feature = "libsql")]
|
||||
async fn step_database_libsql(&mut self) -> Result<(), SetupError> {
|
||||
self.settings.database_backend = Some("libsql".to_string());
|
||||
|
||||
let default_path = crate::config::default_libsql_path();
|
||||
let default_path_str = default_path.to_string_lossy().to_string();
|
||||
|
||||
// Check for existing configuration
|
||||
let existing_path = std::env::var("LIBSQL_PATH")
|
||||
.ok()
|
||||
.or_else(|| self.settings.libsql_path.clone());
|
||||
|
||||
if let Some(ref path) = existing_path {
|
||||
print_info(&format!("Existing database path: {}", path));
|
||||
if confirm("Use this database?", true).map_err(SetupError::Io)? {
|
||||
let turso_url = std::env::var("LIBSQL_URL")
|
||||
.ok()
|
||||
.or_else(|| self.settings.libsql_url.clone());
|
||||
let turso_token = std::env::var("LIBSQL_AUTH_TOKEN").ok();
|
||||
|
||||
match self
|
||||
.test_database_connection_libsql(
|
||||
path,
|
||||
turso_url.as_deref(),
|
||||
turso_token.as_deref(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
print_success("Database connection successful");
|
||||
self.settings.libsql_path = Some(path.clone());
|
||||
if let Some(url) = turso_url {
|
||||
self.settings.libsql_url = Some(url);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
Err(e) => {
|
||||
print_error(&format!("Connection failed: {}", e));
|
||||
print_info("Let's configure a new database path.");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!();
|
||||
print_info("IronClaw uses an embedded SQLite database (libSQL).");
|
||||
print_info("No external database server required.");
|
||||
println!();
|
||||
|
||||
let path_input = optional_input(
|
||||
"Database file path",
|
||||
Some(&format!("default: {}", default_path_str)),
|
||||
)
|
||||
.map_err(SetupError::Io)?;
|
||||
|
||||
let db_path = path_input.unwrap_or(default_path_str.clone());
|
||||
|
||||
// Ask about Turso cloud sync
|
||||
println!();
|
||||
let use_turso =
|
||||
confirm("Enable Turso cloud sync (remote replica)?", false).map_err(SetupError::Io)?;
|
||||
|
||||
let (turso_url, turso_token) = if use_turso {
|
||||
print_info("Enter your Turso database URL and auth token.");
|
||||
print_info("Format: libsql://your-db.turso.io");
|
||||
println!();
|
||||
|
||||
let url = input("Turso URL").map_err(SetupError::Io)?;
|
||||
if url.is_empty() {
|
||||
print_error("Turso URL is required for cloud sync.");
|
||||
(None, None)
|
||||
} else {
|
||||
let token = input("Auth token").map_err(SetupError::Io)?;
|
||||
if token.is_empty() {
|
||||
print_error("Auth token is required for cloud sync.");
|
||||
(None, None)
|
||||
} else {
|
||||
(Some(url), Some(token))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
print_info("Testing connection...");
|
||||
match self
|
||||
.test_database_connection_libsql(&db_path, turso_url.as_deref(), turso_token.as_deref())
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
print_success("Database connection successful");
|
||||
|
||||
// Always run migrations for libsql (they're idempotent)
|
||||
self.run_migrations_libsql().await?;
|
||||
|
||||
self.settings.libsql_path = Some(db_path);
|
||||
if let Some(url) = turso_url {
|
||||
self.settings.libsql_url = Some(url);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => Err(SetupError::Database(format!("Connection failed: {}", e))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Test PostgreSQL connection and store the pool.
|
||||
#[cfg(feature = "postgres")]
|
||||
async fn test_database_connection_postgres(&mut self, url: &str) -> Result<(), SetupError> {
|
||||
let mut cfg = PoolConfig::new();
|
||||
cfg.url = Some(url.to_string());
|
||||
cfg.pool = Some(deadpool_postgres::PoolConfig {
|
||||
@@ -229,7 +374,6 @@ impl SetupWizard {
|
||||
.create_pool(Some(Runtime::Tokio1), NoTls)
|
||||
.map_err(|e| SetupError::Database(format!("Failed to create pool: {}", e)))?;
|
||||
|
||||
// Test the connection
|
||||
let _ = pool
|
||||
.get()
|
||||
.await
|
||||
@@ -239,8 +383,36 @@ impl SetupWizard {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run database migrations.
|
||||
async fn run_migrations(&self) -> Result<(), SetupError> {
|
||||
/// Test libSQL connection and store the backend.
|
||||
#[cfg(feature = "libsql")]
|
||||
async fn test_database_connection_libsql(
|
||||
&mut self,
|
||||
path: &str,
|
||||
turso_url: Option<&str>,
|
||||
turso_token: Option<&str>,
|
||||
) -> Result<(), SetupError> {
|
||||
use crate::db::libsql_backend::LibSqlBackend;
|
||||
use std::path::Path;
|
||||
|
||||
let db_path = Path::new(path);
|
||||
|
||||
let backend = if let (Some(url), Some(token)) = (turso_url, turso_token) {
|
||||
LibSqlBackend::new_remote_replica(db_path, url, token)
|
||||
.await
|
||||
.map_err(|e| SetupError::Database(format!("Failed to connect: {}", e)))?
|
||||
} else {
|
||||
LibSqlBackend::new_local(db_path)
|
||||
.await
|
||||
.map_err(|e| SetupError::Database(format!("Failed to open database: {}", e)))?
|
||||
};
|
||||
|
||||
self.db_backend = Some(backend);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run PostgreSQL migrations.
|
||||
#[cfg(feature = "postgres")]
|
||||
async fn run_migrations_postgres(&self) -> Result<(), SetupError> {
|
||||
if let Some(ref pool) = self.db_pool {
|
||||
use refinery::embed_migrations;
|
||||
embed_migrations!("migrations");
|
||||
@@ -262,6 +434,24 @@ impl SetupWizard {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run libSQL migrations.
|
||||
#[cfg(feature = "libsql")]
|
||||
async fn run_migrations_libsql(&self) -> Result<(), SetupError> {
|
||||
if let Some(ref backend) = self.db_backend {
|
||||
use crate::db::Database;
|
||||
|
||||
print_info("Running migrations...");
|
||||
|
||||
backend
|
||||
.run_migrations()
|
||||
.await
|
||||
.map_err(|e| SetupError::Database(format!("Migration failed: {}", e)))?;
|
||||
|
||||
print_success("Migrations applied");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Step 2: Security (secrets master key).
|
||||
async fn step_security(&mut self) -> Result<(), SetupError> {
|
||||
// Check current configuration
|
||||
@@ -532,24 +722,6 @@ impl SetupWizard {
|
||||
|
||||
/// Initialize secrets context for channel setup.
|
||||
async fn init_secrets_context(&mut self) -> Result<SecretsContext, SetupError> {
|
||||
// Get database pool (should be set from step 1)
|
||||
let pool = if let Some(ref p) = self.db_pool {
|
||||
p.clone()
|
||||
} else {
|
||||
// Fall back to creating one from settings/env
|
||||
let url = self
|
||||
.settings
|
||||
.database_url
|
||||
.clone()
|
||||
.or_else(|| std::env::var("DATABASE_URL").ok())
|
||||
.ok_or_else(|| SetupError::Config("Database URL not configured".to_string()))?;
|
||||
|
||||
self.test_database_connection(&url).await?;
|
||||
// Ensure secrets-related tables exist for channels-only onboarding flows.
|
||||
self.run_migrations().await?;
|
||||
self.db_pool.clone().unwrap()
|
||||
};
|
||||
|
||||
// Get crypto (should be set from step 2, or load from keychain/env)
|
||||
let crypto = if let Some(ref c) = self.secrets_crypto {
|
||||
Arc::clone(c)
|
||||
@@ -571,7 +743,74 @@ impl SetupWizard {
|
||||
Arc::clone(self.secrets_crypto.as_ref().unwrap())
|
||||
};
|
||||
|
||||
Ok(SecretsContext::new(pool, crypto, "default"))
|
||||
// Create backend-appropriate secrets store
|
||||
#[cfg(feature = "postgres")]
|
||||
{
|
||||
// Try postgres path first when postgres feature is available
|
||||
if let Some(store) = self.create_postgres_secrets_store(&crypto).await? {
|
||||
return Ok(SecretsContext::from_store(store, "default"));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
{
|
||||
if let Some(store) = self.create_libsql_secrets_store(&crypto)? {
|
||||
return Ok(SecretsContext::from_store(store, "default"));
|
||||
}
|
||||
}
|
||||
|
||||
Err(SetupError::Config(
|
||||
"No database backend available for secrets storage".to_string(),
|
||||
))
|
||||
}
|
||||
|
||||
/// Create a PostgreSQL secrets store from the current pool.
|
||||
#[cfg(feature = "postgres")]
|
||||
async fn create_postgres_secrets_store(
|
||||
&mut self,
|
||||
crypto: &Arc<SecretsCrypto>,
|
||||
) -> Result<Option<Arc<dyn SecretsStore>>, SetupError> {
|
||||
let pool = if let Some(ref p) = self.db_pool {
|
||||
p.clone()
|
||||
} else {
|
||||
// Fall back to creating one from settings/env
|
||||
let url = self
|
||||
.settings
|
||||
.database_url
|
||||
.clone()
|
||||
.or_else(|| std::env::var("DATABASE_URL").ok());
|
||||
|
||||
if let Some(url) = url {
|
||||
self.test_database_connection_postgres(&url).await?;
|
||||
self.run_migrations_postgres().await?;
|
||||
self.db_pool.clone().unwrap()
|
||||
} else {
|
||||
return Ok(None);
|
||||
}
|
||||
};
|
||||
|
||||
let store: Arc<dyn SecretsStore> = Arc::new(crate::secrets::PostgresSecretsStore::new(
|
||||
pool,
|
||||
Arc::clone(crypto),
|
||||
));
|
||||
Ok(Some(store))
|
||||
}
|
||||
|
||||
/// Create a libSQL secrets store from the current backend.
|
||||
#[cfg(feature = "libsql")]
|
||||
fn create_libsql_secrets_store(
|
||||
&self,
|
||||
crypto: &Arc<SecretsCrypto>,
|
||||
) -> Result<Option<Arc<dyn SecretsStore>>, SetupError> {
|
||||
if let Some(ref backend) = self.db_backend {
|
||||
let store: Arc<dyn SecretsStore> = Arc::new(crate::secrets::LibSqlSecretsStore::new(
|
||||
backend.shared_db(),
|
||||
Arc::clone(crypto),
|
||||
));
|
||||
Ok(Some(store))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
/// Step 6: Channel configuration.
|
||||
@@ -794,8 +1033,27 @@ impl SetupWizard {
|
||||
println!("Configuration Summary:");
|
||||
println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━");
|
||||
|
||||
if self.settings.database_url.is_some() {
|
||||
println!(" Database: configured");
|
||||
let backend = self
|
||||
.settings
|
||||
.database_backend
|
||||
.as_deref()
|
||||
.unwrap_or("postgres");
|
||||
match backend {
|
||||
"libsql" => {
|
||||
if let Some(ref path) = self.settings.libsql_path {
|
||||
println!(" Database: libSQL ({})", path);
|
||||
} else {
|
||||
println!(" Database: libSQL (default path)");
|
||||
}
|
||||
if self.settings.libsql_url.is_some() {
|
||||
println!(" Turso sync: enabled");
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
if self.settings.database_url.is_some() {
|
||||
println!(" Database: PostgreSQL (configured)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
match self.settings.secrets_master_key_source {
|
||||
@@ -875,6 +1133,7 @@ impl Default for SetupWizard {
|
||||
}
|
||||
|
||||
/// Mask password in a database URL for display.
|
||||
#[cfg(feature = "postgres")]
|
||||
fn mask_password_in_url(url: &str) -> String {
|
||||
// URL format: scheme://user:password@host/database
|
||||
// Find "://" to locate start of credentials
|
||||
@@ -1065,6 +1324,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(feature = "postgres")]
|
||||
fn test_mask_password_in_url() {
|
||||
assert_eq!(
|
||||
mask_password_in_url("postgres://user:secret@localhost/db"),
|
||||
@@ -1087,6 +1347,15 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_install_missing_bundled_channels_installs_telegram() {
|
||||
use crate::channels::wasm::available_channel_names;
|
||||
|
||||
// WASM artifacts only exist in dev builds (not CI). Skip gracefully
|
||||
// rather than fail when the telegram channel hasn't been compiled.
|
||||
if !available_channel_names().contains(&"telegram") {
|
||||
eprintln!("skipping: telegram WASM artifacts not built");
|
||||
return;
|
||||
}
|
||||
|
||||
let dir = tempdir().unwrap();
|
||||
let installed = HashSet::<String>::new();
|
||||
|
||||
|
||||
@@ -595,7 +595,7 @@ Create alongside the .wasm file to grant capabilities:
|
||||
AgentToolError::BuilderFailed(format!("LLM response failed: {}", e))
|
||||
})?;
|
||||
|
||||
match result {
|
||||
match result.result {
|
||||
RespondResult::Text(response) => {
|
||||
reason_ctx.messages.push(ChatMessage::assistant(&response));
|
||||
|
||||
|
||||
+137
-35
@@ -50,65 +50,90 @@ const MAX_WRITE_SIZE: usize = 5 * 1024 * 1024;
|
||||
/// Maximum directory listing entries.
|
||||
const MAX_DIR_ENTRIES: usize = 500;
|
||||
|
||||
/// Validate that a path is safe (no traversal attacks).
|
||||
fn validate_path(path_str: &str, base_dir: Option<&Path>) -> Result<PathBuf, ToolError> {
|
||||
let path = PathBuf::from(path_str);
|
||||
|
||||
// Reject paths with suspicious components (validation only, no action needed)
|
||||
/// Normalize a path by resolving `.` and `..` components lexically (no filesystem access).
|
||||
///
|
||||
/// This is critical for security: `std::fs::canonicalize` only works on paths that exist,
|
||||
/// so for new files we must normalize without touching the filesystem.
|
||||
fn normalize_lexical(path: &Path) -> PathBuf {
|
||||
let mut components = Vec::new();
|
||||
for component in path.components() {
|
||||
match component {
|
||||
std::path::Component::ParentDir => {
|
||||
// Allow .. but validate final path is within sandbox
|
||||
}
|
||||
std::path::Component::Normal(s) => {
|
||||
let s = s.to_string_lossy();
|
||||
if s.starts_with('.') && s != "." && s != ".." && !s.starts_with(".git") {
|
||||
// Hidden files are OK for .git, .gitignore, etc.
|
||||
// Only pop if there's a normal component to pop (don't escape root/prefix)
|
||||
if components
|
||||
.last()
|
||||
.is_some_and(|c| matches!(c, std::path::Component::Normal(_)))
|
||||
{
|
||||
components.pop();
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
std::path::Component::CurDir => {}
|
||||
other => components.push(other),
|
||||
}
|
||||
}
|
||||
components.iter().collect()
|
||||
}
|
||||
|
||||
/// Validate that a path is safe (no traversal attacks).
|
||||
///
|
||||
/// For sandboxed paths (base_dir is set), we normalize the joined path lexically
|
||||
/// and then verify it lives under the canonical base. This prevents escapes through
|
||||
/// non-existent parent directories where `canonicalize()` would fall back to the
|
||||
/// raw (un-normalized) path.
|
||||
fn validate_path(path_str: &str, base_dir: Option<&Path>) -> Result<PathBuf, ToolError> {
|
||||
let path = PathBuf::from(path_str);
|
||||
|
||||
// Resolve to absolute path
|
||||
let resolved = if path.is_absolute() {
|
||||
path.canonicalize().unwrap_or_else(|_| path.clone())
|
||||
path.canonicalize()
|
||||
.unwrap_or_else(|_| normalize_lexical(&path))
|
||||
} else if let Some(base) = base_dir {
|
||||
base.join(&path)
|
||||
let joined = base.join(&path);
|
||||
joined
|
||||
.canonicalize()
|
||||
.unwrap_or_else(|_| base.join(&path))
|
||||
.unwrap_or_else(|_| normalize_lexical(&joined))
|
||||
} else {
|
||||
std::env::current_dir()
|
||||
let joined = std::env::current_dir()
|
||||
.unwrap_or_else(|_| PathBuf::from("."))
|
||||
.join(&path)
|
||||
.join(&path);
|
||||
normalize_lexical(&joined)
|
||||
};
|
||||
|
||||
// If base_dir is set, ensure path is within it
|
||||
// If base_dir is set, ensure the resolved path is within it
|
||||
if let Some(base) = base_dir {
|
||||
// Canonicalize the base to handle symlinks (e.g., /var -> /private/var on macOS)
|
||||
let base_canonical = base.canonicalize().unwrap_or_else(|_| base.to_path_buf());
|
||||
let base_canonical = base
|
||||
.canonicalize()
|
||||
.unwrap_or_else(|_| normalize_lexical(base));
|
||||
|
||||
// For files that don't exist yet, we need to check the parent directory
|
||||
// and ensure the resolved path would be within the base
|
||||
// For existing paths, canonicalize to resolve symlinks.
|
||||
// For non-existent paths, the lexical normalization above already removed
|
||||
// all `..` components, so starts_with is reliable.
|
||||
let check_path = if resolved.exists() {
|
||||
resolved.canonicalize().unwrap_or_else(|_| resolved.clone())
|
||||
} else {
|
||||
// For non-existent files, canonicalize the parent and append the filename
|
||||
if let Some(parent) = resolved.parent() {
|
||||
if parent.exists() {
|
||||
let canonical_parent = parent
|
||||
// Walk up to the nearest existing ancestor directory, canonicalize it,
|
||||
// then re-append the remaining tail. This handles the case where a
|
||||
// symlink sits above the new file.
|
||||
let mut ancestor = resolved.as_path();
|
||||
let mut tail_parts: Vec<&std::ffi::OsStr> = Vec::new();
|
||||
loop {
|
||||
if ancestor.exists() {
|
||||
let canonical_ancestor = ancestor
|
||||
.canonicalize()
|
||||
.unwrap_or_else(|_| parent.to_path_buf());
|
||||
if let Some(filename) = resolved.file_name() {
|
||||
canonical_parent.join(filename)
|
||||
} else {
|
||||
resolved.clone()
|
||||
.unwrap_or_else(|_| ancestor.to_path_buf());
|
||||
let mut result = canonical_ancestor;
|
||||
for part in tail_parts.into_iter().rev() {
|
||||
result = result.join(part);
|
||||
}
|
||||
} else {
|
||||
resolved.clone()
|
||||
break result;
|
||||
}
|
||||
if let Some(name) = ancestor.file_name() {
|
||||
tail_parts.push(name);
|
||||
}
|
||||
match ancestor.parent() {
|
||||
Some(parent) if parent != ancestor => ancestor = parent,
|
||||
_ => break resolved.clone(),
|
||||
}
|
||||
} else {
|
||||
resolved.clone()
|
||||
}
|
||||
};
|
||||
|
||||
@@ -871,4 +896,81 @@ mod tests {
|
||||
let entries = result.result.get("entries").unwrap().as_array().unwrap();
|
||||
assert!(entries.len() >= 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_normalize_lexical() {
|
||||
// Basic .. resolution
|
||||
assert_eq!(
|
||||
normalize_lexical(Path::new("/a/b/../c")),
|
||||
PathBuf::from("/a/c")
|
||||
);
|
||||
// Multiple .. components
|
||||
assert_eq!(
|
||||
normalize_lexical(Path::new("/a/b/c/../../d")),
|
||||
PathBuf::from("/a/d")
|
||||
);
|
||||
// . components stripped
|
||||
assert_eq!(
|
||||
normalize_lexical(Path::new("/a/./b/./c")),
|
||||
PathBuf::from("/a/b/c")
|
||||
);
|
||||
// Cannot escape root
|
||||
assert_eq!(
|
||||
normalize_lexical(Path::new("/a/../../..")),
|
||||
PathBuf::from("/")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_path_rejects_traversal_nonexistent_parent() {
|
||||
// The critical test: writing to ../../outside/newdir/file with base_dir
|
||||
// set should be rejected even when the parent directory does not exist
|
||||
// (i.e. canonicalize() cannot resolve it).
|
||||
let dir = TempDir::new().unwrap();
|
||||
let evil_path = format!(
|
||||
"{}/../../outside/newdir/file.txt",
|
||||
dir.path().to_str().unwrap()
|
||||
);
|
||||
let result = validate_path(&evil_path, Some(dir.path()));
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"Should reject traversal via non-existent parent, got: {:?}",
|
||||
result
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_path_rejects_relative_traversal() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let result = validate_path("../../etc/passwd", Some(dir.path()));
|
||||
assert!(
|
||||
result.is_err(),
|
||||
"Should reject relative traversal, got: {:?}",
|
||||
result
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_path_allows_valid_nested_write() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let result = validate_path("subdir/newfile.txt", Some(dir.path()));
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"Should allow nested writes within sandbox: {:?}",
|
||||
result
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_path_allows_dot_dot_within_sandbox() {
|
||||
// a/b/../c resolves to a/c which is still inside the sandbox
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::create_dir_all(dir.path().join("a/b")).unwrap();
|
||||
let result = validate_path("a/b/../c.txt", Some(dir.path()));
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"Should allow .. that stays within sandbox: {:?}",
|
||||
result
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
//! HTTP request tool.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::net::IpAddr;
|
||||
use std::net::{IpAddr, ToSocketAddrs};
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
@@ -11,6 +11,9 @@ use crate::context::JobContext;
|
||||
use crate::safety::LeakDetector;
|
||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
||||
|
||||
/// Maximum response body size (5 MB). Prevents OOM from unbounded responses.
|
||||
const MAX_RESPONSE_SIZE: usize = 5 * 1024 * 1024;
|
||||
|
||||
/// Tool for making HTTP requests.
|
||||
pub struct HttpTool {
|
||||
client: Client,
|
||||
@@ -21,6 +24,7 @@ impl HttpTool {
|
||||
pub fn new() -> Self {
|
||||
let client = Client::builder()
|
||||
.timeout(Duration::from_secs(30))
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.expect("Failed to create HTTP client");
|
||||
|
||||
@@ -49,6 +53,7 @@ fn validate_url(url: &str) -> Result<reqwest::Url, ToolError> {
|
||||
));
|
||||
}
|
||||
|
||||
// Check literal IP addresses
|
||||
if let Ok(ip) = host.parse::<IpAddr>() {
|
||||
if is_disallowed_ip(&ip) {
|
||||
return Err(ToolError::NotAuthorized(
|
||||
@@ -57,6 +62,22 @@ fn validate_url(url: &str) -> Result<reqwest::Url, ToolError> {
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve hostname and check all resolved IPs against the blocklist.
|
||||
// This prevents DNS rebinding where a hostname resolves to a private IP.
|
||||
let port = parsed.port_or_known_default().unwrap_or(443);
|
||||
let socket_addr = format!("{}:{}", host, port);
|
||||
if let Ok(addrs) = socket_addr.to_socket_addrs() {
|
||||
for addr in addrs {
|
||||
if is_disallowed_ip(&addr.ip()) {
|
||||
return Err(ToolError::NotAuthorized(format!(
|
||||
"hostname '{}' resolves to disallowed IP {}",
|
||||
host,
|
||||
addr.ip()
|
||||
)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(parsed)
|
||||
}
|
||||
|
||||
@@ -202,17 +223,36 @@ impl Tool for HttpTool {
|
||||
})?;
|
||||
|
||||
let status = response.status().as_u16();
|
||||
|
||||
// Block redirects: the server tried to send us elsewhere (potential SSRF)
|
||||
if (300..400).contains(&status) {
|
||||
return Err(ToolError::NotAuthorized(format!(
|
||||
"request returned redirect (HTTP {}), which is blocked to prevent SSRF",
|
||||
status
|
||||
)));
|
||||
}
|
||||
|
||||
let headers: HashMap<String, String> = response
|
||||
.headers()
|
||||
.iter()
|
||||
.filter_map(|(k, v)| v.to_str().ok().map(|v| (k.to_string(), v.to_string())))
|
||||
.collect();
|
||||
|
||||
// Get response body
|
||||
let body_text = response.text().await.map_err(|e| {
|
||||
// Get response body with size cap to prevent OOM
|
||||
let body_bytes = response.bytes().await.map_err(|e| {
|
||||
ToolError::ExternalService(format!("failed to read response body: {}", e))
|
||||
})?;
|
||||
|
||||
if body_bytes.len() > MAX_RESPONSE_SIZE {
|
||||
return Err(ToolError::ExecutionFailed(format!(
|
||||
"Response body too large ({} bytes, max {})",
|
||||
body_bytes.len(),
|
||||
MAX_RESPONSE_SIZE
|
||||
)));
|
||||
}
|
||||
|
||||
let body_text = String::from_utf8_lossy(&body_bytes).into_owned();
|
||||
|
||||
// Try to parse as JSON, fall back to string
|
||||
let body: serde_json::Value = serde_json::from_str(&body_text)
|
||||
.unwrap_or_else(|_| serde_json::Value::String(body_text.clone()));
|
||||
@@ -241,7 +281,7 @@ impl Tool for HttpTool {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::validate_url;
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_validate_url_rejects_http() {
|
||||
@@ -260,4 +300,40 @@ mod tests {
|
||||
let url = validate_url("https://example.com").unwrap();
|
||||
assert_eq!(url.host_str(), Some("example.com"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_url_rejects_private_ip_literal() {
|
||||
let err = validate_url("https://192.168.1.1/api").unwrap_err();
|
||||
assert!(err.to_string().contains("private"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_url_rejects_loopback_ip() {
|
||||
let err = validate_url("https://127.0.0.1/api").unwrap_err();
|
||||
assert!(err.to_string().contains("private"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_url_rejects_link_local() {
|
||||
let err = validate_url("https://169.254.169.254/latest/meta-data/").unwrap_err();
|
||||
assert!(err.to_string().contains("private"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_disallowed_ip_covers_ranges() {
|
||||
use std::net::Ipv4Addr;
|
||||
|
||||
// Private ranges
|
||||
assert!(is_disallowed_ip(&IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))));
|
||||
assert!(is_disallowed_ip(&IpAddr::V4(Ipv4Addr::new(172, 16, 0, 1))));
|
||||
assert!(is_disallowed_ip(&IpAddr::V4(Ipv4Addr::new(192, 168, 0, 1))));
|
||||
// Loopback
|
||||
assert!(is_disallowed_ip(&IpAddr::V4(Ipv4Addr::LOCALHOST)));
|
||||
// Cloud metadata
|
||||
assert!(is_disallowed_ip(&IpAddr::V4(Ipv4Addr::new(
|
||||
169, 254, 169, 254
|
||||
))));
|
||||
// Public
|
||||
assert!(!is_disallowed_ip(&IpAddr::V4(Ipv4Addr::new(8, 8, 8, 8))));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,7 +15,8 @@ use chrono::Utc;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::context::{ContextManager, JobContext, JobState};
|
||||
use crate::history::{SandboxJobRecord, Store};
|
||||
use crate::db::Database;
|
||||
use crate::history::SandboxJobRecord;
|
||||
use crate::orchestrator::job_manager::{ContainerJobManager, JobMode};
|
||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
||||
|
||||
@@ -27,7 +28,7 @@ use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
||||
pub struct CreateJobTool {
|
||||
context_manager: Arc<ContextManager>,
|
||||
job_manager: Option<Arc<ContainerJobManager>>,
|
||||
store: Option<Arc<Store>>,
|
||||
store: Option<Arc<dyn Database>>,
|
||||
}
|
||||
|
||||
impl CreateJobTool {
|
||||
@@ -43,7 +44,7 @@ impl CreateJobTool {
|
||||
pub fn with_sandbox(
|
||||
mut self,
|
||||
job_manager: Arc<ContainerJobManager>,
|
||||
store: Option<Arc<Store>>,
|
||||
store: Option<Arc<dyn Database>>,
|
||||
) -> Self {
|
||||
self.job_manager = Some(job_manager);
|
||||
self.store = store;
|
||||
|
||||
@@ -20,6 +20,12 @@ use crate::context::JobContext;
|
||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
||||
use crate::workspace::{Workspace, paths};
|
||||
|
||||
/// Identity files that the LLM must not overwrite via tool calls.
|
||||
/// These are loaded into the system prompt and could be used for prompt
|
||||
/// injection if an attacker tricks the agent into overwriting them.
|
||||
const PROTECTED_IDENTITY_FILES: &[&str] =
|
||||
&[paths::IDENTITY, paths::SOUL, paths::AGENTS, paths::USER];
|
||||
|
||||
/// Tool for searching workspace memory.
|
||||
///
|
||||
/// Performs hybrid search (FTS + semantic) across all memory documents.
|
||||
@@ -188,6 +194,16 @@ impl Tool for MemoryWriteTool {
|
||||
.and_then(|v| v.as_str())
|
||||
.unwrap_or("daily_log");
|
||||
|
||||
// Reject writes to identity files that are loaded into the system prompt.
|
||||
// An attacker could use prompt injection to trick the agent into overwriting
|
||||
// these, poisoning future conversations.
|
||||
if PROTECTED_IDENTITY_FILES.contains(&target) {
|
||||
return Err(ToolError::NotAuthorized(format!(
|
||||
"writing to '{}' is not allowed (identity file protected from tool writes)",
|
||||
target,
|
||||
)));
|
||||
}
|
||||
|
||||
let append = params
|
||||
.get("append")
|
||||
.and_then(|v| v.as_bool())
|
||||
@@ -230,6 +246,20 @@ impl Tool for MemoryWriteTool {
|
||||
paths::HEARTBEAT.to_string()
|
||||
}
|
||||
path => {
|
||||
// Protect identity files from LLM overwrites (prompt injection defense).
|
||||
// These files are injected into the system prompt, so poisoning them
|
||||
// would let an attacker rewrite the agent's core instructions.
|
||||
let normalized = path.trim_start_matches('/');
|
||||
if PROTECTED_IDENTITY_FILES
|
||||
.iter()
|
||||
.any(|p| normalized.eq_ignore_ascii_case(p))
|
||||
{
|
||||
return Err(ToolError::NotAuthorized(format!(
|
||||
"writing to '{}' is not allowed (identity file protected from tool access)",
|
||||
path
|
||||
)));
|
||||
}
|
||||
|
||||
if append {
|
||||
self.workspace
|
||||
.append(path, content)
|
||||
@@ -452,7 +482,7 @@ impl Tool for MemoryTreeTool {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(all(test, feature = "postgres"))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ mod marketplace;
|
||||
mod memory;
|
||||
mod restaurant;
|
||||
pub mod routine;
|
||||
mod shell;
|
||||
pub(crate) mod shell;
|
||||
mod taskrabbit;
|
||||
mod time;
|
||||
|
||||
|
||||
@@ -19,18 +19,18 @@ use crate::agent::routine::{
|
||||
};
|
||||
use crate::agent::routine_engine::RoutineEngine;
|
||||
use crate::context::JobContext;
|
||||
use crate::history::Store;
|
||||
use crate::db::Database;
|
||||
use crate::tools::tool::{Tool, ToolError, ToolOutput};
|
||||
|
||||
// ==================== routine_create ====================
|
||||
|
||||
pub struct RoutineCreateTool {
|
||||
store: Arc<Store>,
|
||||
store: Arc<dyn Database>,
|
||||
engine: Arc<RoutineEngine>,
|
||||
}
|
||||
|
||||
impl RoutineCreateTool {
|
||||
pub fn new(store: Arc<Store>, engine: Arc<RoutineEngine>) -> Self {
|
||||
pub fn new(store: Arc<dyn Database>, engine: Arc<RoutineEngine>) -> Self {
|
||||
Self { store, engine }
|
||||
}
|
||||
}
|
||||
@@ -277,11 +277,11 @@ impl Tool for RoutineCreateTool {
|
||||
// ==================== routine_list ====================
|
||||
|
||||
pub struct RoutineListTool {
|
||||
store: Arc<Store>,
|
||||
store: Arc<dyn Database>,
|
||||
}
|
||||
|
||||
impl RoutineListTool {
|
||||
pub fn new(store: Arc<Store>) -> Self {
|
||||
pub fn new(store: Arc<dyn Database>) -> Self {
|
||||
Self { store }
|
||||
}
|
||||
}
|
||||
@@ -351,12 +351,12 @@ impl Tool for RoutineListTool {
|
||||
// ==================== routine_update ====================
|
||||
|
||||
pub struct RoutineUpdateTool {
|
||||
store: Arc<Store>,
|
||||
store: Arc<dyn Database>,
|
||||
engine: Arc<RoutineEngine>,
|
||||
}
|
||||
|
||||
impl RoutineUpdateTool {
|
||||
pub fn new(store: Arc<Store>, engine: Arc<RoutineEngine>) -> Self {
|
||||
pub fn new(store: Arc<dyn Database>, engine: Arc<RoutineEngine>) -> Self {
|
||||
Self { store, engine }
|
||||
}
|
||||
}
|
||||
@@ -474,12 +474,12 @@ impl Tool for RoutineUpdateTool {
|
||||
// ==================== routine_delete ====================
|
||||
|
||||
pub struct RoutineDeleteTool {
|
||||
store: Arc<Store>,
|
||||
store: Arc<dyn Database>,
|
||||
engine: Arc<RoutineEngine>,
|
||||
}
|
||||
|
||||
impl RoutineDeleteTool {
|
||||
pub fn new(store: Arc<Store>, engine: Arc<RoutineEngine>) -> Self {
|
||||
pub fn new(store: Arc<dyn Database>, engine: Arc<RoutineEngine>) -> Self {
|
||||
Self { store, engine }
|
||||
}
|
||||
}
|
||||
@@ -551,11 +551,11 @@ impl Tool for RoutineDeleteTool {
|
||||
// ==================== routine_history ====================
|
||||
|
||||
pub struct RoutineHistoryTool {
|
||||
store: Arc<Store>,
|
||||
store: Arc<dyn Database>,
|
||||
}
|
||||
|
||||
impl RoutineHistoryTool {
|
||||
pub fn new(store: Arc<Store>) -> Self {
|
||||
pub fn new(store: Arc<dyn Database>) -> Self {
|
||||
Self { store }
|
||||
}
|
||||
}
|
||||
|
||||
+83
-14
@@ -74,6 +74,58 @@ static DANGEROUS_PATTERNS: LazyLock<Vec<&'static str>> = LazyLock::new(|| {
|
||||
]
|
||||
});
|
||||
|
||||
/// Patterns that should NEVER be auto-approved, even if the user chose "always approve"
|
||||
/// for the shell tool. These require explicit per-invocation approval because they are
|
||||
/// destructive or security-sensitive.
|
||||
static NEVER_AUTO_APPROVE_PATTERNS: LazyLock<Vec<&'static str>> = LazyLock::new(|| {
|
||||
vec![
|
||||
"rm -rf",
|
||||
"rm -fr",
|
||||
"chmod -r 777",
|
||||
"chmod 777",
|
||||
"chown -r",
|
||||
"shutdown",
|
||||
"reboot",
|
||||
"poweroff",
|
||||
"init 0",
|
||||
"init 6",
|
||||
"iptables",
|
||||
"nft ",
|
||||
"useradd",
|
||||
"userdel",
|
||||
"passwd",
|
||||
"visudo",
|
||||
"crontab",
|
||||
"systemctl disable",
|
||||
"launchctl unload",
|
||||
"kill -9",
|
||||
"killall",
|
||||
"pkill",
|
||||
"docker rm",
|
||||
"docker rmi",
|
||||
"docker system prune",
|
||||
"git push --force",
|
||||
"git push -f",
|
||||
"git reset --hard",
|
||||
"git clean -f",
|
||||
"DROP TABLE",
|
||||
"DROP DATABASE",
|
||||
"TRUNCATE",
|
||||
"DELETE FROM",
|
||||
]
|
||||
});
|
||||
|
||||
/// Check whether a shell command contains patterns that must never be auto-approved.
|
||||
///
|
||||
/// Even when the user has chosen "always approve" for the shell tool, these commands
|
||||
/// require explicit per-invocation approval because they are destructive.
|
||||
pub fn requires_explicit_approval(command: &str) -> bool {
|
||||
let lower = command.to_lowercase();
|
||||
NEVER_AUTO_APPROVE_PATTERNS
|
||||
.iter()
|
||||
.any(|p| lower.contains(&p.to_lowercase()))
|
||||
}
|
||||
|
||||
/// Shell command execution tool.
|
||||
pub struct ShellTool {
|
||||
/// Working directory for commands (if None, uses job's working dir or cwd).
|
||||
@@ -289,23 +341,17 @@ impl ShellTool {
|
||||
// Determine timeout
|
||||
let timeout_duration = timeout.map(Duration::from_secs).unwrap_or(self.timeout);
|
||||
|
||||
// Try sandbox execution if available
|
||||
// Use sandbox if configured; fail-closed (never silently fall through
|
||||
// to unsandboxed execution when sandbox was intended).
|
||||
if let Some(ref sandbox) = self.sandbox {
|
||||
if sandbox.is_initialized() || sandbox.config().enabled {
|
||||
match self
|
||||
return self
|
||||
.execute_sandboxed(sandbox, cmd, &cwd, timeout_duration)
|
||||
.await
|
||||
{
|
||||
Ok((output, code)) => return Ok((output, code)),
|
||||
Err(e) => {
|
||||
// Log sandbox failure and fall through to direct execution
|
||||
tracing::warn!("Sandbox execution failed, falling back to direct: {}", e);
|
||||
}
|
||||
}
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
// Fallback to direct execution
|
||||
// Only execute directly when no sandbox was configured at all.
|
||||
let (output, code) = self.execute_direct(cmd, &cwd, timeout_duration).await?;
|
||||
Ok((output, code as i64))
|
||||
}
|
||||
@@ -392,17 +438,19 @@ impl Tool for ShellTool {
|
||||
}
|
||||
}
|
||||
|
||||
/// Truncate output to fit within limits.
|
||||
/// Truncate output to fit within limits (UTF-8 safe).
|
||||
fn truncate_output(s: &str) -> String {
|
||||
if s.len() <= MAX_OUTPUT_SIZE {
|
||||
s.to_string()
|
||||
} else {
|
||||
let half = MAX_OUTPUT_SIZE / 2;
|
||||
let head_end = crate::util::floor_char_boundary(s, half);
|
||||
let tail_start = crate::util::floor_char_boundary(s, s.len() - half);
|
||||
format!(
|
||||
"{}\n\n... [truncated {} bytes] ...\n\n{}",
|
||||
&s[..half],
|
||||
&s[..head_end],
|
||||
s.len() - MAX_OUTPUT_SIZE,
|
||||
&s[s.len() - half..]
|
||||
&s[tail_start..]
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -458,6 +506,27 @@ mod tests {
|
||||
assert!(matches!(result, Err(ToolError::Timeout(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_requires_explicit_approval() {
|
||||
// Destructive commands should require explicit approval
|
||||
assert!(requires_explicit_approval("rm -rf /tmp/stuff"));
|
||||
assert!(requires_explicit_approval("git push --force origin main"));
|
||||
assert!(requires_explicit_approval("git reset --hard HEAD~5"));
|
||||
assert!(requires_explicit_approval("docker rm container_name"));
|
||||
assert!(requires_explicit_approval("kill -9 12345"));
|
||||
assert!(requires_explicit_approval("DROP TABLE users;"));
|
||||
|
||||
// Safe commands should not
|
||||
assert!(!requires_explicit_approval("cargo build"));
|
||||
assert!(!requires_explicit_approval("git status"));
|
||||
assert!(!requires_explicit_approval("ls -la"));
|
||||
assert!(!requires_explicit_approval("echo hello"));
|
||||
assert!(!requires_explicit_approval("cat file.txt"));
|
||||
assert!(!requires_explicit_approval(
|
||||
"git push origin feature-branch"
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sandbox_policy_builder() {
|
||||
let tool = ShellTool::new()
|
||||
|
||||
@@ -333,7 +333,7 @@ pub async fn get_mcp_server(name: &str) -> Result<McpServerConfig, ConfigError>
|
||||
///
|
||||
/// Falls back to the disk file if DB has no entry.
|
||||
pub async fn load_mcp_servers_from_db(
|
||||
store: &crate::history::Store,
|
||||
store: &dyn crate::db::Database,
|
||||
user_id: &str,
|
||||
) -> Result<McpServersFile, ConfigError> {
|
||||
match store.get_setting(user_id, "mcp_servers").await {
|
||||
@@ -357,7 +357,7 @@ pub async fn load_mcp_servers_from_db(
|
||||
|
||||
/// Save MCP server configurations to the database settings table.
|
||||
pub async fn save_mcp_servers_to_db(
|
||||
store: &crate::history::Store,
|
||||
store: &dyn crate::db::Database,
|
||||
user_id: &str,
|
||||
config: &McpServersFile,
|
||||
) -> Result<(), ConfigError> {
|
||||
@@ -371,7 +371,7 @@ pub async fn save_mcp_servers_to_db(
|
||||
|
||||
/// Add a new MCP server configuration (DB-backed).
|
||||
pub async fn add_mcp_server_db(
|
||||
store: &crate::history::Store,
|
||||
store: &dyn crate::db::Database,
|
||||
user_id: &str,
|
||||
config: McpServerConfig,
|
||||
) -> Result<(), ConfigError> {
|
||||
@@ -386,7 +386,7 @@ pub async fn add_mcp_server_db(
|
||||
|
||||
/// Remove an MCP server by name (DB-backed).
|
||||
pub async fn remove_mcp_server_db(
|
||||
store: &crate::history::Store,
|
||||
store: &dyn crate::db::Database,
|
||||
user_id: &str,
|
||||
name: &str,
|
||||
) -> Result<(), ConfigError> {
|
||||
|
||||
+114
-5
@@ -6,8 +6,8 @@ use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
use crate::context::ContextManager;
|
||||
use crate::db::Database;
|
||||
use crate::extensions::ExtensionManager;
|
||||
use crate::history::Store;
|
||||
use crate::llm::{LlmProvider, ToolDefinition};
|
||||
use crate::orchestrator::job_manager::ContainerJobManager;
|
||||
use crate::safety::SafetyLayer;
|
||||
@@ -25,9 +25,46 @@ use crate::tools::wasm::{
|
||||
};
|
||||
use crate::workspace::Workspace;
|
||||
|
||||
/// Names of built-in tools that cannot be shadowed by dynamic registrations.
|
||||
/// This prevents a dynamically built or installed tool from replacing a
|
||||
/// security-critical built-in like "shell" or "memory_write".
|
||||
const PROTECTED_TOOL_NAMES: &[&str] = &[
|
||||
"echo",
|
||||
"time",
|
||||
"json",
|
||||
"http",
|
||||
"shell",
|
||||
"read_file",
|
||||
"write_file",
|
||||
"list_dir",
|
||||
"apply_patch",
|
||||
"memory_search",
|
||||
"memory_write",
|
||||
"memory_read",
|
||||
"memory_tree",
|
||||
"create_job",
|
||||
"list_jobs",
|
||||
"job_status",
|
||||
"cancel_job",
|
||||
"build_software",
|
||||
"tool_search",
|
||||
"tool_install",
|
||||
"tool_auth",
|
||||
"tool_activate",
|
||||
"tool_list",
|
||||
"tool_remove",
|
||||
"routine_create",
|
||||
"routine_list",
|
||||
"routine_update",
|
||||
"routine_delete",
|
||||
"routine_history",
|
||||
];
|
||||
|
||||
/// Registry of available tools.
|
||||
pub struct ToolRegistry {
|
||||
tools: RwLock<HashMap<String, Arc<dyn Tool>>>,
|
||||
/// Tracks which names were registered as built-in (protected from shadowing).
|
||||
builtin_names: RwLock<std::collections::HashSet<String>>,
|
||||
}
|
||||
|
||||
impl ToolRegistry {
|
||||
@@ -35,21 +72,35 @@ impl ToolRegistry {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
tools: RwLock::new(HashMap::new()),
|
||||
builtin_names: RwLock::new(std::collections::HashSet::new()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Register a tool.
|
||||
/// Register a tool. Rejects dynamic tools that try to shadow a built-in name.
|
||||
pub async fn register(&self, tool: Arc<dyn Tool>) {
|
||||
let name = tool.name().to_string();
|
||||
if self.builtin_names.read().await.contains(&name) {
|
||||
tracing::warn!(
|
||||
tool = %name,
|
||||
"Rejected tool registration: would shadow a built-in tool"
|
||||
);
|
||||
return;
|
||||
}
|
||||
self.tools.write().await.insert(name.clone(), tool);
|
||||
tracing::debug!("Registered tool: {}", name);
|
||||
}
|
||||
|
||||
/// Register a tool (sync version for startup).
|
||||
/// Register a tool (sync version for startup, marks as built-in).
|
||||
pub fn register_sync(&self, tool: Arc<dyn Tool>) {
|
||||
let name = tool.name().to_string();
|
||||
if let Ok(mut tools) = self.tools.try_write() {
|
||||
tools.insert(name.clone(), tool);
|
||||
// Mark as built-in so it can't be shadowed later
|
||||
if PROTECTED_TOOL_NAMES.contains(&name.as_str()) {
|
||||
if let Ok(mut builtins) = self.builtin_names.try_write() {
|
||||
builtins.insert(name.clone());
|
||||
}
|
||||
}
|
||||
tracing::debug!("Registered tool: {}", name);
|
||||
}
|
||||
}
|
||||
@@ -192,7 +243,7 @@ impl ToolRegistry {
|
||||
&self,
|
||||
context_manager: Arc<ContextManager>,
|
||||
job_manager: Option<Arc<ContainerJobManager>>,
|
||||
store: Option<Arc<Store>>,
|
||||
store: Option<Arc<dyn Database>>,
|
||||
) {
|
||||
let mut create_tool = CreateJobTool::new(Arc::clone(&context_manager));
|
||||
if let Some(jm) = job_manager {
|
||||
@@ -225,7 +276,7 @@ impl ToolRegistry {
|
||||
/// of routines (scheduled and event-driven tasks).
|
||||
pub fn register_routine_tools(
|
||||
&self,
|
||||
store: Arc<Store>,
|
||||
store: Arc<dyn Database>,
|
||||
engine: Arc<crate::agent::routine_engine::RoutineEngine>,
|
||||
) {
|
||||
use crate::tools::builtin::{
|
||||
@@ -419,6 +470,14 @@ impl Default for ToolRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ToolRegistry {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("ToolRegistry")
|
||||
.field("count", &self.count())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -452,4 +511,54 @@ mod tests {
|
||||
assert_eq!(defs.len(), 1);
|
||||
assert_eq!(defs[0].name, "echo");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_builtin_tool_cannot_be_shadowed() {
|
||||
let registry = ToolRegistry::new();
|
||||
// Register echo as built-in (uses register_sync which marks protected names)
|
||||
registry.register_sync(Arc::new(EchoTool));
|
||||
assert!(registry.has("echo").await);
|
||||
|
||||
let original_desc = registry
|
||||
.get("echo")
|
||||
.await
|
||||
.unwrap()
|
||||
.description()
|
||||
.to_string();
|
||||
|
||||
// Create a fake tool that tries to shadow "echo"
|
||||
struct FakeEcho;
|
||||
#[async_trait::async_trait]
|
||||
impl Tool for FakeEcho {
|
||||
fn name(&self) -> &str {
|
||||
"echo"
|
||||
}
|
||||
fn description(&self) -> &str {
|
||||
"EVIL SHADOW"
|
||||
}
|
||||
fn parameters_schema(&self) -> serde_json::Value {
|
||||
serde_json::json!({})
|
||||
}
|
||||
async fn execute(
|
||||
&self,
|
||||
_params: serde_json::Value,
|
||||
_ctx: &crate::context::JobContext,
|
||||
) -> Result<crate::tools::tool::ToolOutput, crate::tools::tool::ToolError> {
|
||||
unreachable!()
|
||||
}
|
||||
}
|
||||
|
||||
// Try to shadow via register() (dynamic path)
|
||||
registry.register(Arc::new(FakeEcho)).await;
|
||||
|
||||
// The original should still be there
|
||||
let desc = registry
|
||||
.get("echo")
|
||||
.await
|
||||
.unwrap()
|
||||
.description()
|
||||
.to_string();
|
||||
assert_eq!(desc, original_desc);
|
||||
assert_ne!(desc, "EVIL SHADOW");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,6 +182,17 @@ fn parse_url(url: &str) -> Result<ParsedUrl, String> {
|
||||
return Err(format!("Unsupported scheme: {}", scheme));
|
||||
}
|
||||
|
||||
// Reject URLs with userinfo (user:pass@host) to prevent allowlist bypass.
|
||||
// A URL like https://[email protected]/ would match the allowlist
|
||||
// for api.openai.com but actually send traffic to evil.com.
|
||||
let authority = match rest.find('/') {
|
||||
Some(idx) => &rest[..idx],
|
||||
None => rest,
|
||||
};
|
||||
if authority.contains('@') {
|
||||
return Err("URL contains userinfo (@) which is not allowed".to_string());
|
||||
}
|
||||
|
||||
// Split host from path
|
||||
let (host_and_port, path) = match rest.find('/') {
|
||||
Some(idx) => (&rest[..idx], &rest[idx..]),
|
||||
@@ -207,6 +218,14 @@ fn parse_url(url: &str) -> Result<ParsedUrl, String> {
|
||||
None => host_and_port,
|
||||
};
|
||||
|
||||
// Reject URLs with userinfo (user:pass@host).
|
||||
// A URL like https://[email protected]/ confuses the parser into
|
||||
// seeing "api.openai.com" as the host, but reqwest actually sends to
|
||||
// "evil.com". Block any '@' in the authority section to prevent this.
|
||||
if host.contains('@') || host_and_port.contains('@') {
|
||||
return Err("URL contains userinfo (@) which is not allowed".to_string());
|
||||
}
|
||||
|
||||
// Validate host
|
||||
if host.is_empty() {
|
||||
return Err("Empty host".to_string());
|
||||
@@ -332,6 +351,21 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_userinfo_rejected() {
|
||||
let validator = validator_with_patterns();
|
||||
|
||||
// Userinfo in URL should be rejected to prevent allowlist bypass
|
||||
let result = validator.validate("https://[email protected]/v1/chat", "GET");
|
||||
assert!(!result.is_allowed());
|
||||
|
||||
if let super::AllowlistResult::Denied(reason) = result {
|
||||
assert!(matches!(reason, DenyReason::InvalidUrl(_)));
|
||||
} else {
|
||||
panic!("Expected denied for userinfo URL");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_invalid_url() {
|
||||
let validator = validator_with_patterns();
|
||||
@@ -354,4 +388,28 @@ mod tests {
|
||||
let result = validator.validate("http://localhost:8080/api", "GET");
|
||||
assert!(result.is_allowed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reject_url_with_userinfo() {
|
||||
let validator = validator_with_patterns();
|
||||
|
||||
// Attacker uses userinfo to trick the parser: the allowlist sees
|
||||
// "api.openai.com" but reqwest would actually connect to "evil.com".
|
||||
let result = validator.validate("https://[email protected]/v1/steal", "GET");
|
||||
assert!(!result.is_allowed());
|
||||
|
||||
if let super::AllowlistResult::Denied(reason) = result {
|
||||
assert!(matches!(reason, DenyReason::InvalidUrl(_)));
|
||||
} else {
|
||||
panic!("Expected denied due to userinfo");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reject_url_with_user_pass() {
|
||||
let validator = validator_with_patterns();
|
||||
|
||||
let result = validator.validate("https://user:[email protected]/v1/chat", "GET");
|
||||
assert!(!result.is_allowed());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,10 +108,13 @@ pub use credential_injector::{CredentialInjector, InjectedCredentials, Injection
|
||||
pub use rate_limiter::{LimitType, RateLimitError, RateLimitResult, RateLimiter};
|
||||
|
||||
// Storage (V2)
|
||||
#[cfg(feature = "libsql")]
|
||||
pub use storage::LibSqlWasmToolStore;
|
||||
#[cfg(feature = "postgres")]
|
||||
pub use storage::PostgresWasmToolStore;
|
||||
pub use storage::{
|
||||
PostgresWasmToolStore, StoreToolParams, StoredCapabilities, StoredWasmTool,
|
||||
StoredWasmToolWithBinary, ToolStatus, TrustLevel, WasmStorageError, WasmToolStore,
|
||||
compute_binary_hash, verify_binary_integrity,
|
||||
StoreToolParams, StoredCapabilities, StoredWasmTool, StoredWasmToolWithBinary, ToolStatus,
|
||||
TrustLevel, WasmStorageError, WasmToolStore, compute_binary_hash, verify_binary_integrity,
|
||||
};
|
||||
|
||||
// Loader
|
||||
|
||||
@@ -14,6 +14,10 @@ use wasmtime::{Config, Engine, OptLevel};
|
||||
use crate::tools::wasm::error::WasmError;
|
||||
use crate::tools::wasm::limits::{FuelConfig, ResourceLimits};
|
||||
|
||||
/// Default epoch tick interval. Each tick increments the engine's epoch counter,
|
||||
/// which causes any store with an expired epoch deadline to trap.
|
||||
pub const EPOCH_TICK_INTERVAL: Duration = Duration::from_millis(500);
|
||||
|
||||
/// Configuration for the WASM runtime.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct WasmRuntimeConfig {
|
||||
@@ -123,6 +127,25 @@ impl WasmToolRuntime {
|
||||
WasmError::EngineCreationFailed(format!("Failed to create Wasmtime engine: {}", e))
|
||||
})?;
|
||||
|
||||
// Spawn a background thread that periodically increments the engine's
|
||||
// epoch counter. Without this, epoch_deadline_trap() never fires and
|
||||
// WASM modules can spin indefinitely even with a deadline set.
|
||||
let ticker_engine = engine.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("wasm-epoch-ticker".into())
|
||||
.spawn(move || {
|
||||
loop {
|
||||
std::thread::sleep(EPOCH_TICK_INTERVAL);
|
||||
ticker_engine.increment_epoch();
|
||||
}
|
||||
})
|
||||
.map_err(|e| {
|
||||
WasmError::EngineCreationFailed(format!(
|
||||
"Failed to spawn epoch ticker thread: {}",
|
||||
e
|
||||
))
|
||||
})?;
|
||||
|
||||
Ok(Self {
|
||||
engine,
|
||||
config,
|
||||
|
||||
@@ -16,6 +16,7 @@ use std::collections::HashMap;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use chrono::{DateTime, Utc};
|
||||
#[cfg(feature = "postgres")]
|
||||
use deadpool_postgres::Pool;
|
||||
use uuid::Uuid;
|
||||
|
||||
@@ -263,16 +264,19 @@ pub fn verify_binary_integrity(binary: &[u8], expected_hash: &[u8]) -> bool {
|
||||
}
|
||||
|
||||
/// PostgreSQL implementation of WasmToolStore.
|
||||
#[cfg(feature = "postgres")]
|
||||
pub struct PostgresWasmToolStore {
|
||||
pool: Pool,
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
impl PostgresWasmToolStore {
|
||||
pub fn new(pool: Pool) -> Self {
|
||||
Self { pool }
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
#[async_trait]
|
||||
impl WasmToolStore for PostgresWasmToolStore {
|
||||
async fn store(&self, params: StoreToolParams) -> Result<StoredWasmTool, WasmStorageError> {
|
||||
@@ -538,6 +542,7 @@ impl WasmToolStore for PostgresWasmToolStore {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "postgres")]
|
||||
fn row_to_tool(row: &tokio_postgres::Row) -> Result<StoredWasmTool, WasmStorageError> {
|
||||
let trust_level_str: String = row.get("trust_level");
|
||||
let status_str: String = row.get("status");
|
||||
@@ -559,6 +564,458 @@ fn row_to_tool(row: &tokio_postgres::Row) -> Result<StoredWasmTool, WasmStorageE
|
||||
})
|
||||
}
|
||||
|
||||
// ==================== libSQL implementation ====================
|
||||
|
||||
/// libSQL/Turso implementation of WasmToolStore.
|
||||
///
|
||||
/// Holds an `Arc<Database>` handle and creates a fresh connection per operation,
|
||||
/// matching the connection-per-request pattern used by the main `LibSqlBackend`.
|
||||
#[cfg(feature = "libsql")]
|
||||
pub struct LibSqlWasmToolStore {
|
||||
db: std::sync::Arc<libsql::Database>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
impl LibSqlWasmToolStore {
|
||||
pub fn new(db: std::sync::Arc<libsql::Database>) -> Self {
|
||||
Self { db }
|
||||
}
|
||||
|
||||
fn connect(&self) -> Result<libsql::Connection, WasmStorageError> {
|
||||
self.db
|
||||
.connect()
|
||||
.map_err(|e| WasmStorageError::Database(format!("Connection failed: {}", e)))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
#[async_trait]
|
||||
impl WasmToolStore for LibSqlWasmToolStore {
|
||||
async fn store(&self, params: StoreToolParams) -> Result<StoredWasmTool, WasmStorageError> {
|
||||
let binary_hash = compute_binary_hash(¶ms.wasm_binary);
|
||||
let id = Uuid::new_v4();
|
||||
let now = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true);
|
||||
let schema_str = serde_json::to_string(¶ms.parameters_schema)
|
||||
.map_err(|e| WasmStorageError::InvalidData(e.to_string()))?;
|
||||
|
||||
// Wrap INSERT + read-back in a transaction to prevent TOCTOU races
|
||||
let conn = self.connect()?;
|
||||
let tx = conn
|
||||
.transaction()
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
|
||||
tx.execute(
|
||||
r#"
|
||||
INSERT INTO wasm_tools (
|
||||
id, user_id, name, version, description, wasm_binary, binary_hash,
|
||||
parameters_schema, source_url, trust_level, status, created_at, updated_at
|
||||
)
|
||||
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 'active', ?11, ?11)
|
||||
ON CONFLICT (user_id, name, version) DO UPDATE SET
|
||||
description = excluded.description,
|
||||
wasm_binary = excluded.wasm_binary,
|
||||
binary_hash = excluded.binary_hash,
|
||||
parameters_schema = excluded.parameters_schema,
|
||||
source_url = excluded.source_url,
|
||||
updated_at = ?11
|
||||
"#,
|
||||
libsql::params![
|
||||
id.to_string(),
|
||||
params.user_id.as_str(),
|
||||
params.name.as_str(),
|
||||
params.version.as_str(),
|
||||
params.description.as_str(),
|
||||
libsql::Value::Blob(params.wasm_binary),
|
||||
libsql::Value::Blob(binary_hash),
|
||||
schema_str.as_str(),
|
||||
libsql_wasm_opt_text(params.source_url.as_deref()),
|
||||
params.trust_level.to_string(),
|
||||
now.as_str(),
|
||||
],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
|
||||
// Read back the row within the same transaction
|
||||
let mut rows = tx
|
||||
.query(
|
||||
r#"
|
||||
SELECT id, user_id, name, version, description, parameters_schema,
|
||||
source_url, trust_level, status, created_at, updated_at
|
||||
FROM wasm_tools
|
||||
WHERE user_id = ?1 AND name = ?2
|
||||
ORDER BY version DESC
|
||||
LIMIT 1
|
||||
"#,
|
||||
libsql::params![params.user_id.as_str(), params.name.as_str()],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
|
||||
let row = rows
|
||||
.next()
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?
|
||||
.ok_or_else(|| {
|
||||
WasmStorageError::Database("Insert succeeded but row not found".into())
|
||||
})?;
|
||||
|
||||
let tool = libsql_row_to_tool(&row)?;
|
||||
|
||||
tx.commit()
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
|
||||
Ok(tool)
|
||||
}
|
||||
|
||||
async fn get(&self, user_id: &str, name: &str) -> Result<StoredWasmTool, WasmStorageError> {
|
||||
let conn = self.connect()?;
|
||||
let mut rows = conn
|
||||
.query(
|
||||
r#"
|
||||
SELECT id, user_id, name, version, description, parameters_schema,
|
||||
source_url, trust_level, status, created_at, updated_at
|
||||
FROM wasm_tools
|
||||
WHERE user_id = ?1 AND name = ?2 AND status = 'active'
|
||||
ORDER BY version DESC
|
||||
LIMIT 1
|
||||
"#,
|
||||
libsql::params![user_id, name],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
|
||||
match rows
|
||||
.next()
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?
|
||||
{
|
||||
Some(row) => {
|
||||
let tool = libsql_row_to_tool(&row)?;
|
||||
match tool.status {
|
||||
ToolStatus::Active => Ok(tool),
|
||||
ToolStatus::Disabled => Err(WasmStorageError::Disabled),
|
||||
ToolStatus::Quarantined => Err(WasmStorageError::Quarantined),
|
||||
}
|
||||
}
|
||||
None => Err(WasmStorageError::NotFound(name.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_with_binary(
|
||||
&self,
|
||||
user_id: &str,
|
||||
name: &str,
|
||||
) -> Result<StoredWasmToolWithBinary, WasmStorageError> {
|
||||
let conn = self.connect()?;
|
||||
let mut rows = conn
|
||||
.query(
|
||||
r#"
|
||||
SELECT id, user_id, name, version, description, wasm_binary, binary_hash,
|
||||
parameters_schema, source_url, trust_level, status, created_at, updated_at
|
||||
FROM wasm_tools
|
||||
WHERE user_id = ?1 AND name = ?2 AND status = 'active'
|
||||
ORDER BY version DESC
|
||||
LIMIT 1
|
||||
"#,
|
||||
libsql::params![user_id, name],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
|
||||
match rows
|
||||
.next()
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?
|
||||
{
|
||||
Some(row) => {
|
||||
let wasm_binary: Vec<u8> = row
|
||||
.get(5)
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
let binary_hash: Vec<u8> = row
|
||||
.get(6)
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
|
||||
if !verify_binary_integrity(&wasm_binary, &binary_hash) {
|
||||
tracing::error!(
|
||||
user_id = user_id,
|
||||
name = name,
|
||||
"WASM binary integrity check failed"
|
||||
);
|
||||
return Err(WasmStorageError::IntegrityCheckFailed);
|
||||
}
|
||||
|
||||
// Parse metadata from the row (different column offsets due to binary/hash)
|
||||
let tool = libsql_row_to_tool_with_offset(&row)?;
|
||||
|
||||
match tool.status {
|
||||
ToolStatus::Active => Ok(StoredWasmToolWithBinary {
|
||||
tool,
|
||||
wasm_binary,
|
||||
binary_hash,
|
||||
}),
|
||||
ToolStatus::Disabled => Err(WasmStorageError::Disabled),
|
||||
ToolStatus::Quarantined => Err(WasmStorageError::Quarantined),
|
||||
}
|
||||
}
|
||||
None => Err(WasmStorageError::NotFound(name.to_string())),
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_capabilities(
|
||||
&self,
|
||||
tool_id: Uuid,
|
||||
) -> Result<Option<StoredCapabilities>, WasmStorageError> {
|
||||
let conn = self.connect()?;
|
||||
let mut rows = conn
|
||||
.query(
|
||||
r#"
|
||||
SELECT id, wasm_tool_id, http_allowlist, allowed_secrets, tool_aliases,
|
||||
requests_per_minute, requests_per_hour, max_request_body_bytes,
|
||||
max_response_body_bytes, workspace_read_prefixes, http_timeout_secs
|
||||
FROM tool_capabilities
|
||||
WHERE wasm_tool_id = ?1
|
||||
"#,
|
||||
libsql::params![tool_id.to_string()],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
|
||||
match rows
|
||||
.next()
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?
|
||||
{
|
||||
Some(row) => {
|
||||
let id_str: String = row
|
||||
.get(0)
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
let tool_id_str: String = row
|
||||
.get(1)
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
let http_allowlist_str: String = row.get::<String>(2).unwrap_or_default();
|
||||
let allowed_secrets_str: String = row.get::<String>(3).unwrap_or_default();
|
||||
let tool_aliases_str: String = row.get::<String>(4).unwrap_or_default();
|
||||
let rpm: i64 = row.get::<i64>(5).unwrap_or(60);
|
||||
let rph: i64 = row.get::<i64>(6).unwrap_or(1000);
|
||||
let max_req: i64 = row.get::<i64>(7).unwrap_or(1048576);
|
||||
let max_resp: i64 = row.get::<i64>(8).unwrap_or(10485760);
|
||||
let ws_prefixes_str: String = row.get::<String>(9).unwrap_or_default();
|
||||
let timeout: i64 = row.get::<i64>(10).unwrap_or(30);
|
||||
|
||||
let http_allowlist: Vec<EndpointPattern> =
|
||||
serde_json::from_str(&http_allowlist_str).unwrap_or_default();
|
||||
let allowed_secrets: Vec<String> =
|
||||
serde_json::from_str(&allowed_secrets_str).unwrap_or_default();
|
||||
let tool_aliases: HashMap<String, String> =
|
||||
serde_json::from_str(&tool_aliases_str).unwrap_or_default();
|
||||
let workspace_read_prefixes: Vec<String> =
|
||||
serde_json::from_str(&ws_prefixes_str).unwrap_or_default();
|
||||
|
||||
Ok(Some(StoredCapabilities {
|
||||
id: id_str
|
||||
.parse()
|
||||
.map_err(|e: uuid::Error| WasmStorageError::InvalidData(e.to_string()))?,
|
||||
wasm_tool_id: tool_id_str
|
||||
.parse()
|
||||
.map_err(|e: uuid::Error| WasmStorageError::InvalidData(e.to_string()))?,
|
||||
http_allowlist,
|
||||
allowed_secrets,
|
||||
tool_aliases,
|
||||
requests_per_minute: rpm as u32,
|
||||
requests_per_hour: rph as u32,
|
||||
max_request_body_bytes: max_req,
|
||||
max_response_body_bytes: max_resp,
|
||||
workspace_read_prefixes,
|
||||
http_timeout_secs: timeout as i32,
|
||||
}))
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
async fn list(&self, user_id: &str) -> Result<Vec<StoredWasmTool>, WasmStorageError> {
|
||||
// SQLite doesn't have DISTINCT ON, so we use a subquery to get latest version per name
|
||||
let conn = self.connect()?;
|
||||
let mut rows = conn
|
||||
.query(
|
||||
r#"
|
||||
SELECT id, user_id, name, version, description, parameters_schema,
|
||||
source_url, trust_level, status, created_at, updated_at
|
||||
FROM wasm_tools
|
||||
WHERE user_id = ?1
|
||||
AND rowid IN (
|
||||
SELECT MAX(rowid)
|
||||
FROM wasm_tools
|
||||
WHERE user_id = ?1
|
||||
GROUP BY name
|
||||
)
|
||||
ORDER BY name
|
||||
"#,
|
||||
libsql::params![user_id],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
|
||||
let mut tools = Vec::new();
|
||||
while let Some(row) = rows
|
||||
.next()
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?
|
||||
{
|
||||
tools.push(libsql_row_to_tool(&row)?);
|
||||
}
|
||||
Ok(tools)
|
||||
}
|
||||
|
||||
async fn update_status(
|
||||
&self,
|
||||
user_id: &str,
|
||||
name: &str,
|
||||
status: ToolStatus,
|
||||
) -> Result<(), WasmStorageError> {
|
||||
let now = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true);
|
||||
let conn = self.connect()?;
|
||||
|
||||
let result = conn
|
||||
.execute(
|
||||
"UPDATE wasm_tools SET status = ?1, updated_at = ?2 WHERE user_id = ?3 AND name = ?4",
|
||||
libsql::params![status.to_string(), now.as_str(), user_id, name],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
|
||||
if result == 0 {
|
||||
return Err(WasmStorageError::NotFound(name.to_string()));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete(&self, user_id: &str, name: &str) -> Result<bool, WasmStorageError> {
|
||||
let conn = self.connect()?;
|
||||
let result = conn
|
||||
.execute(
|
||||
"DELETE FROM wasm_tools WHERE user_id = ?1 AND name = ?2",
|
||||
libsql::params![user_id, name],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
|
||||
Ok(result > 0)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
fn libsql_wasm_opt_text(s: Option<&str>) -> libsql::Value {
|
||||
match s {
|
||||
Some(s) => libsql::Value::Text(s.to_string()),
|
||||
None => libsql::Value::Null,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
fn libsql_wasm_parse_ts(s: &str) -> Result<DateTime<Utc>, WasmStorageError> {
|
||||
if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(s) {
|
||||
return Ok(dt.with_timezone(&Utc));
|
||||
}
|
||||
if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f") {
|
||||
return Ok(ndt.and_utc());
|
||||
}
|
||||
if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S") {
|
||||
return Ok(ndt.and_utc());
|
||||
}
|
||||
Err(WasmStorageError::InvalidData(format!(
|
||||
"unparseable timestamp: {:?}",
|
||||
s
|
||||
)))
|
||||
}
|
||||
|
||||
/// Parse a tool row with standard column order (no binary columns).
|
||||
/// Columns: id(0), user_id(1), name(2), version(3), description(4),
|
||||
/// parameters_schema(5), source_url(6), trust_level(7), status(8),
|
||||
/// created_at(9), updated_at(10)
|
||||
#[cfg(feature = "libsql")]
|
||||
fn libsql_row_to_tool(row: &libsql::Row) -> Result<StoredWasmTool, WasmStorageError> {
|
||||
libsql_row_to_tool_at(row, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
|
||||
}
|
||||
|
||||
/// Parse a tool row when binary columns are present (get_with_binary query).
|
||||
/// Columns: id(0), user_id(1), name(2), version(3), description(4),
|
||||
/// wasm_binary(5), binary_hash(6),
|
||||
/// parameters_schema(7), source_url(8), trust_level(9), status(10),
|
||||
/// created_at(11), updated_at(12)
|
||||
#[cfg(feature = "libsql")]
|
||||
fn libsql_row_to_tool_with_offset(row: &libsql::Row) -> Result<StoredWasmTool, WasmStorageError> {
|
||||
libsql_row_to_tool_at(row, 0, 1, 2, 3, 4, 7, 8, 9, 10, 11, 12)
|
||||
}
|
||||
|
||||
#[cfg(feature = "libsql")]
|
||||
fn libsql_row_to_tool_at(
|
||||
row: &libsql::Row,
|
||||
id_idx: i32,
|
||||
user_id_idx: i32,
|
||||
name_idx: i32,
|
||||
version_idx: i32,
|
||||
description_idx: i32,
|
||||
schema_idx: i32,
|
||||
source_url_idx: i32,
|
||||
trust_level_idx: i32,
|
||||
status_idx: i32,
|
||||
created_at_idx: i32,
|
||||
updated_at_idx: i32,
|
||||
) -> Result<StoredWasmTool, WasmStorageError> {
|
||||
let id_str: String = row
|
||||
.get(id_idx)
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
let trust_level_str: String = row
|
||||
.get(trust_level_idx)
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
let status_str: String = row
|
||||
.get(status_idx)
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
let schema_str: String = row
|
||||
.get(schema_idx)
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
let created_at_str: String = row
|
||||
.get(created_at_idx)
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
let updated_at_str: String = row
|
||||
.get(updated_at_idx)
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?;
|
||||
|
||||
Ok(StoredWasmTool {
|
||||
id: id_str
|
||||
.parse()
|
||||
.map_err(|e: uuid::Error| WasmStorageError::InvalidData(e.to_string()))?,
|
||||
user_id: row
|
||||
.get(user_id_idx)
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?,
|
||||
name: row
|
||||
.get(name_idx)
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?,
|
||||
version: row
|
||||
.get(version_idx)
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?,
|
||||
description: row
|
||||
.get(description_idx)
|
||||
.map_err(|e| WasmStorageError::Database(e.to_string()))?,
|
||||
parameters_schema: serde_json::from_str(&schema_str).unwrap_or_default(),
|
||||
source_url: row
|
||||
.get::<String>(source_url_idx)
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty()),
|
||||
trust_level: trust_level_str
|
||||
.parse()
|
||||
.map_err(WasmStorageError::InvalidData)?,
|
||||
status: status_str.parse().map_err(WasmStorageError::InvalidData)?,
|
||||
created_at: libsql_wasm_parse_ts(&created_at_str)?,
|
||||
updated_at: libsql_wasm_parse_ts(&updated_at_str)?,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::tools::wasm::storage::{
|
||||
|
||||
+190
-6
@@ -23,7 +23,7 @@ use crate::tools::wasm::capabilities::Capabilities;
|
||||
use crate::tools::wasm::error::WasmError;
|
||||
use crate::tools::wasm::host::{HostState, LogLevel};
|
||||
use crate::tools::wasm::limits::{ResourceLimits, WasmResourceLimiter};
|
||||
use crate::tools::wasm::runtime::{PreparedModule, WasmToolRuntime};
|
||||
use crate::tools::wasm::runtime::{EPOCH_TICK_INTERVAL, PreparedModule, WasmToolRuntime};
|
||||
|
||||
// Generate component model bindings from the WIT file.
|
||||
//
|
||||
@@ -194,10 +194,25 @@ impl near::agent::host::Host for StoreData {
|
||||
.scan_http_request(&url, &header_vec, body.as_deref())
|
||||
.map_err(|e| format!("Potential secret leak blocked: {}", e))?;
|
||||
|
||||
// Get the max response size from capabilities (default 10MB).
|
||||
let max_response_bytes = self
|
||||
.host_state
|
||||
.capabilities()
|
||||
.http
|
||||
.as_ref()
|
||||
.map(|h| h.max_response_bytes)
|
||||
.unwrap_or(10 * 1024 * 1024);
|
||||
|
||||
// Resolve hostname and reject private/internal IPs to prevent DNS rebinding.
|
||||
reject_private_ip(&url)?;
|
||||
|
||||
// Make HTTP request using blocking I/O.
|
||||
// We're inside a spawn_blocking context, so use block_on.
|
||||
let result = tokio::runtime::Handle::current().block_on(async {
|
||||
let client = reqwest::Client::new();
|
||||
let client = reqwest::Client::builder()
|
||||
.redirect(reqwest::redirect::Policy::none())
|
||||
.build()
|
||||
.map_err(|e| format!("failed to create HTTP client: {e}"))?;
|
||||
|
||||
let mut request = match method.to_uppercase().as_str() {
|
||||
"GET" => client.get(&url),
|
||||
@@ -241,11 +256,31 @@ impl near::agent::host::Host for StoreData {
|
||||
})
|
||||
.collect();
|
||||
let headers_json = serde_json::to_string(&response_headers).unwrap_or_default();
|
||||
|
||||
// Check Content-Length header for early rejection of oversized responses.
|
||||
let max_response = max_response_bytes;
|
||||
if let Some(cl) = response.content_length() {
|
||||
if cl as usize > max_response {
|
||||
return Err(format!(
|
||||
"Response body too large: {} bytes exceeds limit of {} bytes",
|
||||
cl, max_response
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// Read body with a size cap to prevent memory exhaustion.
|
||||
let body = response
|
||||
.bytes()
|
||||
.await
|
||||
.map_err(|e| format!("Failed to read response body: {}", e))?
|
||||
.to_vec();
|
||||
.map_err(|e| format!("Failed to read response body: {}", e))?;
|
||||
if body.len() > max_response {
|
||||
return Err(format!(
|
||||
"Response body too large: {} bytes exceeds limit of {} bytes",
|
||||
body.len(),
|
||||
max_response
|
||||
));
|
||||
}
|
||||
let body = body.to_vec();
|
||||
|
||||
// Leak detection on response body
|
||||
if let Ok(body_str) = std::str::from_utf8(&body) {
|
||||
@@ -380,9 +415,13 @@ impl WasmToolWrapper {
|
||||
.map_err(|e| WasmError::ConfigError(format!("Failed to set fuel: {}", e)))?;
|
||||
}
|
||||
|
||||
// Configure epoch deadline for timeout backup
|
||||
// Configure epoch deadline as a hard timeout backup.
|
||||
// The epoch ticker thread increments the engine epoch every EPOCH_TICK_INTERVAL.
|
||||
// Setting deadline to N means "trap after N ticks", so we compute the number
|
||||
// of ticks that fit in the tool's timeout. Minimum 1 to always have a backstop.
|
||||
store.epoch_deadline_trap();
|
||||
store.set_epoch_deadline(1);
|
||||
let ticks = (limits.timeout.as_millis() / EPOCH_TICK_INTERVAL.as_millis()).max(1) as u64;
|
||||
store.set_epoch_deadline(ticks);
|
||||
|
||||
// Set up resource limiter
|
||||
store.limiter(|data| &mut data.limiter);
|
||||
@@ -531,6 +570,88 @@ impl std::fmt::Debug for WasmToolWrapper {
|
||||
}
|
||||
}
|
||||
|
||||
/// Resolve the URL's hostname and reject connections to private/internal IP addresses.
|
||||
/// This prevents DNS rebinding attacks where an attacker's domain resolves to an
|
||||
/// internal IP after passing the allowlist check.
|
||||
fn reject_private_ip(url: &str) -> Result<(), String> {
|
||||
let host = url
|
||||
.split("://")
|
||||
.nth(1)
|
||||
.and_then(|rest| {
|
||||
let host_and_port = rest.split('/').next().unwrap_or(rest);
|
||||
// Strip port
|
||||
if host_and_port.starts_with('[') {
|
||||
// IPv6
|
||||
host_and_port.find(']').map(|i| &host_and_port[1..i])
|
||||
} else {
|
||||
Some(
|
||||
host_and_port
|
||||
.rfind(':')
|
||||
.map_or(host_and_port, |i| &host_and_port[..i]),
|
||||
)
|
||||
}
|
||||
})
|
||||
.ok_or_else(|| "Failed to parse host from URL".to_string())?;
|
||||
|
||||
// If the host is already an IP, check it directly
|
||||
if let Ok(ip) = host.parse::<std::net::IpAddr>() {
|
||||
return if is_private_ip(ip) {
|
||||
Err(format!(
|
||||
"HTTP request to private/internal IP {} is not allowed",
|
||||
ip
|
||||
))
|
||||
} else {
|
||||
Ok(())
|
||||
};
|
||||
}
|
||||
|
||||
// Resolve DNS and check all addresses
|
||||
use std::net::ToSocketAddrs;
|
||||
// Port 0 is a placeholder; ToSocketAddrs needs host:port but the port
|
||||
// doesn't affect which IPs the hostname resolves to.
|
||||
let addrs: Vec<_> = format!("{}:0", host)
|
||||
.to_socket_addrs()
|
||||
.map_err(|e| format!("DNS resolution failed for {}: {}", host, e))?
|
||||
.collect();
|
||||
|
||||
if addrs.is_empty() {
|
||||
return Err(format!("DNS resolution returned no addresses for {}", host));
|
||||
}
|
||||
|
||||
for addr in &addrs {
|
||||
if is_private_ip(addr.ip()) {
|
||||
return Err(format!(
|
||||
"DNS rebinding detected: {} resolved to private IP {}",
|
||||
host,
|
||||
addr.ip()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if an IP address belongs to a private/internal range.
|
||||
fn is_private_ip(ip: std::net::IpAddr) -> bool {
|
||||
match ip {
|
||||
std::net::IpAddr::V4(v4) => {
|
||||
v4.is_loopback() // 127.0.0.0/8
|
||||
|| v4.is_private() // 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16
|
||||
|| v4.is_link_local() // 169.254.0.0/16
|
||||
|| v4.is_unspecified() // 0.0.0.0
|
||||
|| v4.octets()[0] == 100 && (v4.octets()[1] & 0xC0) == 64 // 100.64.0.0/10 (CGNAT)
|
||||
}
|
||||
std::net::IpAddr::V6(v6) => {
|
||||
v6.is_loopback() // ::1
|
||||
|| v6.is_unspecified() // ::
|
||||
// fc00::/7 (unique local)
|
||||
|| (v6.segments()[0] & 0xFE00) == 0xFC00
|
||||
// fe80::/10 (link-local)
|
||||
|| (v6.segments()[0] & 0xFFC0) == 0xFE80
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
@@ -557,4 +678,67 @@ mod tests {
|
||||
assert!(caps.tool_invoke.is_none());
|
||||
assert!(caps.secrets.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_private_ip_v4() {
|
||||
use std::net::IpAddr;
|
||||
// Private ranges
|
||||
assert!(super::is_private_ip("127.0.0.1".parse::<IpAddr>().unwrap()));
|
||||
assert!(super::is_private_ip("10.0.0.1".parse::<IpAddr>().unwrap()));
|
||||
assert!(super::is_private_ip(
|
||||
"172.16.0.1".parse::<IpAddr>().unwrap()
|
||||
));
|
||||
assert!(super::is_private_ip(
|
||||
"192.168.1.1".parse::<IpAddr>().unwrap()
|
||||
));
|
||||
assert!(super::is_private_ip(
|
||||
"169.254.1.1".parse::<IpAddr>().unwrap()
|
||||
));
|
||||
assert!(super::is_private_ip("0.0.0.0".parse::<IpAddr>().unwrap()));
|
||||
// CGNAT
|
||||
assert!(super::is_private_ip(
|
||||
"100.64.0.1".parse::<IpAddr>().unwrap()
|
||||
));
|
||||
|
||||
// Public IPs
|
||||
assert!(!super::is_private_ip("8.8.8.8".parse::<IpAddr>().unwrap()));
|
||||
assert!(!super::is_private_ip("1.1.1.1".parse::<IpAddr>().unwrap()));
|
||||
assert!(!super::is_private_ip(
|
||||
"93.184.216.34".parse::<IpAddr>().unwrap()
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_is_private_ip_v6() {
|
||||
use std::net::IpAddr;
|
||||
assert!(super::is_private_ip("::1".parse::<IpAddr>().unwrap()));
|
||||
assert!(super::is_private_ip("::".parse::<IpAddr>().unwrap()));
|
||||
assert!(super::is_private_ip("fc00::1".parse::<IpAddr>().unwrap()));
|
||||
assert!(super::is_private_ip("fe80::1".parse::<IpAddr>().unwrap()));
|
||||
|
||||
// Public
|
||||
assert!(!super::is_private_ip(
|
||||
"2606:4700::1111".parse::<IpAddr>().unwrap()
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reject_private_ip_loopback() {
|
||||
let result = super::reject_private_ip("https://127.0.0.1:8080/api");
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().contains("private/internal IP"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reject_private_ip_internal() {
|
||||
let result = super::reject_private_ip("https://192.168.1.1/admin");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reject_private_ip_public_ok() {
|
||||
// 8.8.8.8 (Google DNS) is public
|
||||
let result = super::reject_private_ip("https://8.8.8.8/dns-query");
|
||||
assert!(result.is_ok());
|
||||
}
|
||||
}
|
||||
|
||||
+178
@@ -0,0 +1,178 @@
|
||||
//! Shared utility functions used across the codebase.
|
||||
|
||||
/// Find the largest valid UTF-8 char boundary at or before `pos`.
|
||||
///
|
||||
/// Polyfill for `str::floor_char_boundary` (nightly-only). Use when
|
||||
/// truncating strings by byte position to avoid panicking on multi-byte
|
||||
/// characters.
|
||||
pub fn floor_char_boundary(s: &str, pos: usize) -> usize {
|
||||
if pos >= s.len() {
|
||||
return s.len();
|
||||
}
|
||||
let mut i = pos;
|
||||
while i > 0 && !s.is_char_boundary(i) {
|
||||
i -= 1;
|
||||
}
|
||||
i
|
||||
}
|
||||
|
||||
/// Check if an LLM response explicitly signals that a job/task is complete.
|
||||
///
|
||||
/// Uses phrase-level matching to avoid false positives from bare words like
|
||||
/// "done" or "complete" appearing in non-completion contexts (e.g. "not done yet",
|
||||
/// "the download is incomplete").
|
||||
pub fn llm_signals_completion(response: &str) -> bool {
|
||||
let lower = response.to_lowercase();
|
||||
|
||||
// Superset of phrases from agent/worker.rs and worker/runtime.rs.
|
||||
let positive_phrases = [
|
||||
"job is complete",
|
||||
"job is done",
|
||||
"job is finished",
|
||||
"task is complete",
|
||||
"task is done",
|
||||
"task is finished",
|
||||
"work is complete",
|
||||
"work is done",
|
||||
"work is finished",
|
||||
"successfully completed",
|
||||
"have completed the job",
|
||||
"have completed the task",
|
||||
"have finished the job",
|
||||
"have finished the task",
|
||||
"all steps are complete",
|
||||
"all steps are done",
|
||||
"i have completed",
|
||||
"i've completed",
|
||||
"all done",
|
||||
"all tasks complete",
|
||||
];
|
||||
|
||||
let negative_phrases = [
|
||||
"not complete",
|
||||
"not done",
|
||||
"not finished",
|
||||
"incomplete",
|
||||
"unfinished",
|
||||
"isn't done",
|
||||
"isn't complete",
|
||||
"isn't finished",
|
||||
"not yet done",
|
||||
"not yet complete",
|
||||
"not yet finished",
|
||||
];
|
||||
|
||||
let has_negative = negative_phrases.iter().any(|p| lower.contains(p));
|
||||
if has_negative {
|
||||
return false;
|
||||
}
|
||||
|
||||
positive_phrases.iter().any(|p| lower.contains(p))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::util::{floor_char_boundary, llm_signals_completion};
|
||||
|
||||
// ── floor_char_boundary ──
|
||||
|
||||
#[test]
|
||||
fn floor_char_boundary_at_valid_boundary() {
|
||||
assert_eq!(floor_char_boundary("hello", 3), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn floor_char_boundary_mid_multibyte_char() {
|
||||
// h = 1 byte, é = 2 bytes, total 3 bytes
|
||||
let s = "hé";
|
||||
assert_eq!(floor_char_boundary(s, 2), 1); // byte 2 is mid-é, back up to 1
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn floor_char_boundary_past_end() {
|
||||
assert_eq!(floor_char_boundary("hi", 100), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn floor_char_boundary_at_zero() {
|
||||
assert_eq!(floor_char_boundary("hello", 0), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn floor_char_boundary_empty_string() {
|
||||
assert_eq!(floor_char_boundary("", 5), 0);
|
||||
}
|
||||
|
||||
// ── llm_signals_completion ──
|
||||
|
||||
#[test]
|
||||
fn signals_completion_positive() {
|
||||
assert!(llm_signals_completion("The job is complete."));
|
||||
assert!(llm_signals_completion("I have completed the task."));
|
||||
assert!(llm_signals_completion("All done, here are the results."));
|
||||
assert!(llm_signals_completion("Task is finished successfully."));
|
||||
assert!(llm_signals_completion(
|
||||
"I have completed the task successfully."
|
||||
));
|
||||
assert!(llm_signals_completion(
|
||||
"All steps are complete and verified."
|
||||
));
|
||||
assert!(llm_signals_completion(
|
||||
"I've done all the work. The work is done."
|
||||
));
|
||||
assert!(llm_signals_completion(
|
||||
"Successfully completed the migration."
|
||||
));
|
||||
assert!(llm_signals_completion(
|
||||
"I have completed the job ahead of schedule."
|
||||
));
|
||||
assert!(llm_signals_completion("I have finished the task."));
|
||||
assert!(llm_signals_completion("All steps are done now."));
|
||||
assert!(llm_signals_completion("I've completed everything."));
|
||||
assert!(llm_signals_completion("All tasks complete."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signals_completion_negative() {
|
||||
assert!(!llm_signals_completion("The task is not complete yet."));
|
||||
assert!(!llm_signals_completion("This is not done."));
|
||||
assert!(!llm_signals_completion("The work is incomplete."));
|
||||
assert!(!llm_signals_completion("Build is unfinished."));
|
||||
assert!(!llm_signals_completion(
|
||||
"The migration is not yet finished."
|
||||
));
|
||||
assert!(!llm_signals_completion("The job isn't done yet."));
|
||||
assert!(!llm_signals_completion("This remains unfinished."));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signals_completion_no_bare_substrings() {
|
||||
assert!(!llm_signals_completion("The download completed."));
|
||||
assert!(!llm_signals_completion(
|
||||
"Function done_callback was called."
|
||||
));
|
||||
assert!(!llm_signals_completion("Set is_complete = true"));
|
||||
assert!(!llm_signals_completion("Running step 3 of 5"));
|
||||
assert!(!llm_signals_completion(
|
||||
"I need to complete more work first."
|
||||
));
|
||||
assert!(!llm_signals_completion(
|
||||
"Let me finish the remaining steps."
|
||||
));
|
||||
assert!(!llm_signals_completion(
|
||||
"I'm done analyzing, now let me fix it."
|
||||
));
|
||||
assert!(!llm_signals_completion(
|
||||
"I completed step 1 but step 2 remains."
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn signals_completion_tool_output_injection() {
|
||||
assert!(!llm_signals_completion("TASK_COMPLETE"));
|
||||
assert!(!llm_signals_completion("JOB_DONE"));
|
||||
assert!(!llm_signals_completion(
|
||||
"The tool returned: TASK_COMPLETE signal"
|
||||
));
|
||||
}
|
||||
}
|
||||
+106
-12
@@ -4,18 +4,26 @@
|
||||
//! output back to the orchestrator via HTTP. Supports follow-up prompts via
|
||||
//! `--resume`.
|
||||
//!
|
||||
//! Security model: the Docker container is the primary security boundary
|
||||
//! (cap-drop ALL, non-root user, memory limits, network isolation).
|
||||
//! As defense-in-depth, a project-level `.claude/settings.json` is written
|
||||
//! before spawning with an explicit tool allowlist. Only listed tools are
|
||||
//! auto-approved; unknown/future tools would require interactive approval,
|
||||
//! which times out harmlessly in the non-interactive container.
|
||||
//!
|
||||
//! ```text
|
||||
//! ┌─────────────────────────────────────────────┐
|
||||
//! │ Docker Container │
|
||||
//! │ │
|
||||
//! │ ironclaw claude-bridge --job-id <uuid> │
|
||||
//! ┌──────────────────────────────────────────────┐
|
||||
//! │ Docker Container │
|
||||
//! │ │
|
||||
//! │ ironclaw claude-bridge --job-id <uuid> │
|
||||
//! │ └─ writes /workspace/.claude/settings.json │
|
||||
//! │ └─ claude -p "task" --output-format │
|
||||
//! │ stream-json --dangerously-skip-perms │
|
||||
//! │ └─ reads stdout line-by-line │
|
||||
//! │ └─ POSTs events to orchestrator │
|
||||
//! │ └─ polls for follow-up prompts │
|
||||
//! │ └─ on follow-up: claude --resume │
|
||||
//! └─────────────────────────────────────────────┘
|
||||
//! │ stream-json │
|
||||
//! │ └─ reads stdout line-by-line │
|
||||
//! │ └─ POSTs events to orchestrator │
|
||||
//! │ └─ polls for follow-up prompts │
|
||||
//! │ └─ on follow-up: claude --resume │
|
||||
//! └──────────────────────────────────────────────┘
|
||||
//! ```
|
||||
|
||||
use std::sync::Arc;
|
||||
@@ -36,6 +44,8 @@ pub struct ClaudeBridgeConfig {
|
||||
pub max_turns: u32,
|
||||
pub model: String,
|
||||
pub timeout: Duration,
|
||||
/// Tool patterns to auto-approve via project-level settings.json.
|
||||
pub allowed_tools: Vec<String>,
|
||||
}
|
||||
|
||||
/// A Claude Code streaming event (NDJSON line from `--output-format stream-json`).
|
||||
@@ -119,8 +129,37 @@ impl ClaudeBridgeRuntime {
|
||||
Ok(Self { config, client })
|
||||
}
|
||||
|
||||
/// Write project-level `.claude/settings.json` with the tool allowlist.
|
||||
///
|
||||
/// This replaces `--dangerously-skip-permissions` with an explicit set of
|
||||
/// auto-approved tools. The Docker container is still the primary security
|
||||
/// boundary; this is defense-in-depth.
|
||||
fn write_permission_settings(&self) -> Result<(), WorkerError> {
|
||||
let settings_json = build_permission_settings(&self.config.allowed_tools);
|
||||
let settings_dir = std::path::Path::new("/workspace/.claude");
|
||||
std::fs::create_dir_all(settings_dir).map_err(|e| WorkerError::ExecutionFailed {
|
||||
reason: format!("failed to create /workspace/.claude/: {e}"),
|
||||
})?;
|
||||
std::fs::write(settings_dir.join("settings.json"), &settings_json).map_err(|e| {
|
||||
WorkerError::ExecutionFailed {
|
||||
reason: format!("failed to write settings.json: {e}"),
|
||||
}
|
||||
})?;
|
||||
tracing::info!(
|
||||
job_id = %self.config.job_id,
|
||||
tools = ?self.config.allowed_tools,
|
||||
"Wrote Claude Code permission settings"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run the bridge: fetch job, spawn claude, stream events, handle follow-ups.
|
||||
pub async fn run(&self) -> Result<(), WorkerError> {
|
||||
// Write project-level settings with explicit tool allowlist.
|
||||
// This replaces --dangerously-skip-permissions with defense-in-depth:
|
||||
// only the listed tools are auto-approved, unknown tools fail safely.
|
||||
self.write_permission_settings()?;
|
||||
|
||||
// Fetch the job description from the orchestrator
|
||||
let job = self.client.get_job().await?;
|
||||
|
||||
@@ -226,7 +265,6 @@ impl ClaudeBridgeRuntime {
|
||||
.arg(prompt)
|
||||
.arg("--output-format")
|
||||
.arg("stream-json")
|
||||
.arg("--dangerously-skip-permissions")
|
||||
.arg("--max-turns")
|
||||
.arg(self.config.max_turns.to_string())
|
||||
.arg("--model")
|
||||
@@ -380,6 +418,19 @@ impl ClaudeBridgeRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
/// Build the JSON content for `.claude/settings.json` with the given tool allowlist.
|
||||
///
|
||||
/// Produces a Claude Code project settings file that auto-approves the listed
|
||||
/// tools while leaving any unknown/future tools unapproved (defense-in-depth).
|
||||
fn build_permission_settings(allowed_tools: &[String]) -> String {
|
||||
let settings = serde_json::json!({
|
||||
"permissions": {
|
||||
"allow": allowed_tools,
|
||||
}
|
||||
});
|
||||
serde_json::to_string_pretty(&settings).expect("static JSON structure is always valid")
|
||||
}
|
||||
|
||||
/// Convert a Claude stream event into one or more event payloads for the orchestrator.
|
||||
fn stream_event_to_payloads(event: &ClaudeStreamEvent) -> Vec<JobEventPayload> {
|
||||
let mut payloads = Vec::new();
|
||||
@@ -465,7 +516,16 @@ fn stream_event_to_payloads(event: &ClaudeStreamEvent) -> Vec<JobEventPayload> {
|
||||
}
|
||||
|
||||
fn truncate(s: &str, max_len: usize) -> &str {
|
||||
if s.len() <= max_len { s } else { &s[..max_len] }
|
||||
if s.len() <= max_len {
|
||||
s
|
||||
} else {
|
||||
// Walk back from max_len to find a valid UTF-8 char boundary.
|
||||
let mut end = max_len;
|
||||
while end > 0 && !s.is_char_boundary(end) {
|
||||
end -= 1;
|
||||
}
|
||||
&s[..end]
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -641,4 +701,38 @@ mod tests {
|
||||
assert_eq!(truncate("hello world", 5), "hello");
|
||||
assert_eq!(truncate("", 5), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_permission_settings_default_tools() {
|
||||
let tools: Vec<String> = ["Bash(*)", "Read", "Edit(*)", "Glob", "Grep"]
|
||||
.into_iter()
|
||||
.map(String::from)
|
||||
.collect();
|
||||
let json_str = build_permission_settings(&tools);
|
||||
let parsed: serde_json::Value = serde_json::from_str(&json_str).unwrap();
|
||||
let allow = parsed["permissions"]["allow"].as_array().unwrap();
|
||||
assert_eq!(allow.len(), 5);
|
||||
assert_eq!(allow[0], "Bash(*)");
|
||||
assert_eq!(allow[1], "Read");
|
||||
assert_eq!(allow[2], "Edit(*)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_permission_settings_empty_tools() {
|
||||
let json_str = build_permission_settings(&[]);
|
||||
let parsed: serde_json::Value = serde_json::from_str(&json_str).unwrap();
|
||||
let allow = parsed["permissions"]["allow"].as_array().unwrap();
|
||||
assert!(allow.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_build_permission_settings_is_valid_json() {
|
||||
let tools = vec!["Bash(npm run *)".to_string(), "Read".to_string()];
|
||||
let json_str = build_permission_settings(&tools);
|
||||
// Must be valid JSON
|
||||
let parsed: serde_json::Value = serde_json::from_str(&json_str).unwrap();
|
||||
// Must have the expected structure
|
||||
assert!(parsed["permissions"].is_object());
|
||||
assert!(parsed["permissions"]["allow"].is_array());
|
||||
}
|
||||
}
|
||||
|
||||
+38
-8
@@ -238,7 +238,7 @@ Work independently to complete this job. Report when done."#,
|
||||
reason: format!("respond_with_tools failed: {}", e),
|
||||
})?;
|
||||
|
||||
match respond_result {
|
||||
match respond_result.result {
|
||||
RespondResult::Text(response) => {
|
||||
self.post_event(
|
||||
"message",
|
||||
@@ -249,11 +249,7 @@ Work independently to complete this job. Report when done."#,
|
||||
)
|
||||
.await;
|
||||
|
||||
let response_lower = response.to_lowercase();
|
||||
if response_lower.contains("complete")
|
||||
|| response_lower.contains("finished")
|
||||
|| response_lower.contains("done")
|
||||
{
|
||||
if crate::util::llm_signals_completion(&response) {
|
||||
if last_output.is_empty() {
|
||||
last_output = response.clone();
|
||||
}
|
||||
@@ -431,7 +427,11 @@ Work independently to complete this job. Report when done."#,
|
||||
wrapped,
|
||||
));
|
||||
|
||||
output.contains("TASK_COMPLETE") || output.contains("JOB_DONE")
|
||||
// Tool output should never signal job completion. Only the LLM's
|
||||
// natural language response should decide when a job is done. A
|
||||
// tool could return text containing "TASK_COMPLETE" in its output
|
||||
// (e.g. from file contents) and trigger a false positive.
|
||||
false
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Tool {} failed: {}", selection.tool_name, e);
|
||||
@@ -486,6 +486,36 @@ fn truncate(s: &str, max: usize) -> String {
|
||||
if s.len() <= max {
|
||||
s.to_string()
|
||||
} else {
|
||||
format!("{}...", &s[..max])
|
||||
let end = crate::util::floor_char_boundary(s, max);
|
||||
format!("{}...", &s[..end])
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::worker::runtime::truncate;
|
||||
|
||||
#[test]
|
||||
fn test_truncate_within_limit() {
|
||||
assert_eq!(truncate("hello", 10), "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_truncate_at_limit() {
|
||||
assert_eq!(truncate("hello", 5), "hello");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_truncate_beyond_limit() {
|
||||
let result = truncate("hello world", 5);
|
||||
assert_eq!(result, "hello...");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_truncate_multibyte_safe() {
|
||||
// "é" is 2 bytes in UTF-8; slicing at byte 1 would panic without safety
|
||||
let result = truncate("é is fancy", 1);
|
||||
// Should truncate to 0 chars (can't fit "é" in 1 byte)
|
||||
assert_eq!(result, "...");
|
||||
}
|
||||
}
|
||||
|
||||
+223
-24
@@ -43,23 +43,206 @@
|
||||
mod chunker;
|
||||
mod document;
|
||||
mod embeddings;
|
||||
#[cfg(feature = "postgres")]
|
||||
mod repository;
|
||||
mod search;
|
||||
|
||||
pub use chunker::{ChunkConfig, chunk_document};
|
||||
pub use document::{MemoryChunk, MemoryDocument, WorkspaceEntry, paths};
|
||||
pub use embeddings::{EmbeddingProvider, MockEmbeddings, NearAiEmbeddings, OpenAiEmbeddings};
|
||||
#[cfg(feature = "postgres")]
|
||||
pub use repository::Repository;
|
||||
pub use search::{SearchConfig, SearchResult};
|
||||
pub use search::{RankedResult, SearchConfig, SearchResult, reciprocal_rank_fusion};
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use chrono::{NaiveDate, Utc};
|
||||
#[cfg(feature = "postgres")]
|
||||
use deadpool_postgres::Pool;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::error::WorkspaceError;
|
||||
|
||||
/// Internal storage abstraction for Workspace.
|
||||
///
|
||||
/// Allows Workspace to work with either a PostgreSQL `Repository` (the original
|
||||
/// path) or any `Database` trait implementation (e.g. libSQL backend).
|
||||
enum WorkspaceStorage {
|
||||
/// PostgreSQL-backed repository (uses connection pool directly).
|
||||
#[cfg(feature = "postgres")]
|
||||
Repo(Repository),
|
||||
/// Generic backend implementing the Database trait.
|
||||
Db(Arc<dyn crate::db::Database>),
|
||||
}
|
||||
|
||||
impl WorkspaceStorage {
|
||||
async fn get_document_by_path(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
path: &str,
|
||||
) -> Result<MemoryDocument, WorkspaceError> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Repo(repo) => repo.get_document_by_path(user_id, agent_id, path).await,
|
||||
Self::Db(db) => db.get_document_by_path(user_id, agent_id, path).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_document_by_id(&self, id: Uuid) -> Result<MemoryDocument, WorkspaceError> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Repo(repo) => repo.get_document_by_id(id).await,
|
||||
Self::Db(db) => db.get_document_by_id(id).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_or_create_document_by_path(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
path: &str,
|
||||
) -> Result<MemoryDocument, WorkspaceError> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Repo(repo) => {
|
||||
repo.get_or_create_document_by_path(user_id, agent_id, path)
|
||||
.await
|
||||
}
|
||||
Self::Db(db) => {
|
||||
db.get_or_create_document_by_path(user_id, agent_id, path)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn update_document(&self, id: Uuid, content: &str) -> Result<(), WorkspaceError> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Repo(repo) => repo.update_document(id, content).await,
|
||||
Self::Db(db) => db.update_document(id, content).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_document_by_path(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
path: &str,
|
||||
) -> Result<(), WorkspaceError> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Repo(repo) => repo.delete_document_by_path(user_id, agent_id, path).await,
|
||||
Self::Db(db) => db.delete_document_by_path(user_id, agent_id, path).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_directory(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
directory: &str,
|
||||
) -> Result<Vec<WorkspaceEntry>, WorkspaceError> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Repo(repo) => repo.list_directory(user_id, agent_id, directory).await,
|
||||
Self::Db(db) => db.list_directory(user_id, agent_id, directory).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_all_paths(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
) -> Result<Vec<String>, WorkspaceError> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Repo(repo) => repo.list_all_paths(user_id, agent_id).await,
|
||||
Self::Db(db) => db.list_all_paths(user_id, agent_id).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn delete_chunks(&self, document_id: Uuid) -> Result<(), WorkspaceError> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Repo(repo) => repo.delete_chunks(document_id).await,
|
||||
Self::Db(db) => db.delete_chunks(document_id).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn insert_chunk(
|
||||
&self,
|
||||
document_id: Uuid,
|
||||
chunk_index: i32,
|
||||
content: &str,
|
||||
embedding: Option<&[f32]>,
|
||||
) -> Result<Uuid, WorkspaceError> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Repo(repo) => {
|
||||
repo.insert_chunk(document_id, chunk_index, content, embedding)
|
||||
.await
|
||||
}
|
||||
Self::Db(db) => {
|
||||
db.insert_chunk(document_id, chunk_index, content, embedding)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn update_chunk_embedding(
|
||||
&self,
|
||||
chunk_id: Uuid,
|
||||
embedding: &[f32],
|
||||
) -> Result<(), WorkspaceError> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Repo(repo) => repo.update_chunk_embedding(chunk_id, embedding).await,
|
||||
Self::Db(db) => db.update_chunk_embedding(chunk_id, embedding).await,
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_chunks_without_embeddings(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
limit: usize,
|
||||
) -> Result<Vec<MemoryChunk>, WorkspaceError> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Repo(repo) => {
|
||||
repo.get_chunks_without_embeddings(user_id, agent_id, limit)
|
||||
.await
|
||||
}
|
||||
Self::Db(db) => {
|
||||
db.get_chunks_without_embeddings(user_id, agent_id, limit)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn hybrid_search(
|
||||
&self,
|
||||
user_id: &str,
|
||||
agent_id: Option<Uuid>,
|
||||
query: &str,
|
||||
embedding: Option<&[f32]>,
|
||||
config: &SearchConfig,
|
||||
) -> Result<Vec<SearchResult>, WorkspaceError> {
|
||||
match self {
|
||||
#[cfg(feature = "postgres")]
|
||||
Self::Repo(repo) => {
|
||||
repo.hybrid_search(user_id, agent_id, query, embedding, config)
|
||||
.await
|
||||
}
|
||||
Self::Db(db) => {
|
||||
db.hybrid_search(user_id, agent_id, query, embedding, config)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Default template seeded into HEARTBEAT.md on first access.
|
||||
///
|
||||
/// Intentionally comment-only so the heartbeat runner treats it as
|
||||
@@ -80,25 +263,39 @@ const HEARTBEAT_SEED: &str = "\
|
||||
/// Workspace provides database-backed memory storage for an agent.
|
||||
///
|
||||
/// Each workspace is scoped to a user (and optionally an agent).
|
||||
/// Documents are persisted to PostgreSQL and indexed for search.
|
||||
/// Documents are persisted to the database and indexed for search.
|
||||
/// Supports both PostgreSQL (via Repository) and libSQL (via Database trait).
|
||||
pub struct Workspace {
|
||||
/// User identifier (from channel).
|
||||
user_id: String,
|
||||
/// Optional agent ID for multi-agent isolation.
|
||||
agent_id: Option<Uuid>,
|
||||
/// Database repository.
|
||||
repo: Repository,
|
||||
/// Database storage backend.
|
||||
storage: WorkspaceStorage,
|
||||
/// Embedding provider for semantic search.
|
||||
embeddings: Option<Arc<dyn EmbeddingProvider>>,
|
||||
}
|
||||
|
||||
impl Workspace {
|
||||
/// Create a new workspace for a user.
|
||||
/// Create a new workspace backed by a PostgreSQL connection pool.
|
||||
#[cfg(feature = "postgres")]
|
||||
pub fn new(user_id: impl Into<String>, pool: Pool) -> Self {
|
||||
Self {
|
||||
user_id: user_id.into(),
|
||||
agent_id: None,
|
||||
repo: Repository::new(pool),
|
||||
storage: WorkspaceStorage::Repo(Repository::new(pool)),
|
||||
embeddings: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new workspace backed by any Database implementation.
|
||||
///
|
||||
/// Use this for libSQL or any other backend that implements the Database trait.
|
||||
pub fn new_with_db(user_id: impl Into<String>, db: Arc<dyn crate::db::Database>) -> Self {
|
||||
Self {
|
||||
user_id: user_id.into(),
|
||||
agent_id: None,
|
||||
storage: WorkspaceStorage::Db(db),
|
||||
embeddings: None,
|
||||
}
|
||||
}
|
||||
@@ -138,7 +335,7 @@ impl Workspace {
|
||||
/// ```
|
||||
pub async fn read(&self, path: &str) -> Result<MemoryDocument, WorkspaceError> {
|
||||
let path = normalize_path(path);
|
||||
self.repo
|
||||
self.storage
|
||||
.get_document_by_path(&self.user_id, self.agent_id, &path)
|
||||
.await
|
||||
}
|
||||
@@ -155,14 +352,14 @@ impl Workspace {
|
||||
pub async fn write(&self, path: &str, content: &str) -> Result<MemoryDocument, WorkspaceError> {
|
||||
let path = normalize_path(path);
|
||||
let doc = self
|
||||
.repo
|
||||
.storage
|
||||
.get_or_create_document_by_path(&self.user_id, self.agent_id, &path)
|
||||
.await?;
|
||||
self.repo.update_document(doc.id, content).await?;
|
||||
self.storage.update_document(doc.id, content).await?;
|
||||
self.reindex_document(doc.id).await?;
|
||||
|
||||
// Return updated doc
|
||||
self.repo.get_document_by_id(doc.id).await
|
||||
self.storage.get_document_by_id(doc.id).await
|
||||
}
|
||||
|
||||
/// Append content to a file.
|
||||
@@ -172,7 +369,7 @@ impl Workspace {
|
||||
pub async fn append(&self, path: &str, content: &str) -> Result<(), WorkspaceError> {
|
||||
let path = normalize_path(path);
|
||||
let doc = self
|
||||
.repo
|
||||
.storage
|
||||
.get_or_create_document_by_path(&self.user_id, self.agent_id, &path)
|
||||
.await?;
|
||||
|
||||
@@ -182,7 +379,7 @@ impl Workspace {
|
||||
format!("{}\n{}", doc.content, content)
|
||||
};
|
||||
|
||||
self.repo.update_document(doc.id, &new_content).await?;
|
||||
self.storage.update_document(doc.id, &new_content).await?;
|
||||
self.reindex_document(doc.id).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -191,7 +388,7 @@ impl Workspace {
|
||||
pub async fn exists(&self, path: &str) -> Result<bool, WorkspaceError> {
|
||||
let path = normalize_path(path);
|
||||
match self
|
||||
.repo
|
||||
.storage
|
||||
.get_document_by_path(&self.user_id, self.agent_id, &path)
|
||||
.await
|
||||
{
|
||||
@@ -206,7 +403,7 @@ impl Workspace {
|
||||
/// Also deletes associated chunks.
|
||||
pub async fn delete(&self, path: &str) -> Result<(), WorkspaceError> {
|
||||
let path = normalize_path(path);
|
||||
self.repo
|
||||
self.storage
|
||||
.delete_document_by_path(&self.user_id, self.agent_id, &path)
|
||||
.await
|
||||
}
|
||||
@@ -229,14 +426,16 @@ impl Workspace {
|
||||
/// ```
|
||||
pub async fn list(&self, directory: &str) -> Result<Vec<WorkspaceEntry>, WorkspaceError> {
|
||||
let directory = normalize_directory(directory);
|
||||
self.repo
|
||||
self.storage
|
||||
.list_directory(&self.user_id, self.agent_id, &directory)
|
||||
.await
|
||||
}
|
||||
|
||||
/// List all files recursively (flat list of all paths).
|
||||
pub async fn list_all(&self) -> Result<Vec<String>, WorkspaceError> {
|
||||
self.repo.list_all_paths(&self.user_id, self.agent_id).await
|
||||
self.storage
|
||||
.list_all_paths(&self.user_id, self.agent_id)
|
||||
.await
|
||||
}
|
||||
|
||||
// ==================== Convenience Methods ====================
|
||||
@@ -280,7 +479,7 @@ impl Workspace {
|
||||
|
||||
/// Helper to read or create a file.
|
||||
async fn read_or_create(&self, path: &str) -> Result<MemoryDocument, WorkspaceError> {
|
||||
self.repo
|
||||
self.storage
|
||||
.get_or_create_document_by_path(&self.user_id, self.agent_id, path)
|
||||
.await
|
||||
}
|
||||
@@ -299,7 +498,7 @@ impl Workspace {
|
||||
} else {
|
||||
format!("{}\n\n{}", doc.content, entry)
|
||||
};
|
||||
self.repo.update_document(doc.id, &new_content).await?;
|
||||
self.storage.update_document(doc.id, &new_content).await?;
|
||||
self.reindex_document(doc.id).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -395,7 +594,7 @@ impl Workspace {
|
||||
None
|
||||
};
|
||||
|
||||
self.repo
|
||||
self.storage
|
||||
.hybrid_search(
|
||||
&self.user_id,
|
||||
self.agent_id,
|
||||
@@ -411,13 +610,13 @@ impl Workspace {
|
||||
/// Re-index a document (chunk and generate embeddings).
|
||||
async fn reindex_document(&self, document_id: Uuid) -> Result<(), WorkspaceError> {
|
||||
// Get the document
|
||||
let doc = self.repo.get_document_by_id(document_id).await?;
|
||||
let doc = self.storage.get_document_by_id(document_id).await?;
|
||||
|
||||
// Chunk the content
|
||||
let chunks = chunk_document(&doc.content, ChunkConfig::default());
|
||||
|
||||
// Delete old chunks
|
||||
self.repo.delete_chunks(document_id).await?;
|
||||
self.storage.delete_chunks(document_id).await?;
|
||||
|
||||
// Insert new chunks
|
||||
for (index, content) in chunks.into_iter().enumerate() {
|
||||
@@ -434,7 +633,7 @@ impl Workspace {
|
||||
None
|
||||
};
|
||||
|
||||
self.repo
|
||||
self.storage
|
||||
.insert_chunk(document_id, index as i32, &content, embedding.as_deref())
|
||||
.await?;
|
||||
}
|
||||
@@ -542,7 +741,7 @@ impl Workspace {
|
||||
};
|
||||
|
||||
let chunks = self
|
||||
.repo
|
||||
.storage
|
||||
.get_chunks_without_embeddings(&self.user_id, self.agent_id, 100)
|
||||
.await?;
|
||||
|
||||
@@ -550,7 +749,7 @@ impl Workspace {
|
||||
for chunk in chunks {
|
||||
match provider.embed(&chunk.content).await {
|
||||
Ok(embedding) => {
|
||||
self.repo
|
||||
self.storage
|
||||
.update_chunk_embedding(chunk.id, &embedding)
|
||||
.await?;
|
||||
count += 1;
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
//! Integration tests for the OpenAI-compatible API endpoints.
|
||||
//!
|
||||
//! Uses a mock LLM provider so no real API key is needed.
|
||||
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use rust_decimal::Decimal;
|
||||
|
||||
use ironclaw::channels::web::server::{GatewayState, start_server};
|
||||
use ironclaw::channels::web::sse::SseManager;
|
||||
use ironclaw::channels::web::ws::WsConnectionTracker;
|
||||
use ironclaw::error::LlmError;
|
||||
use ironclaw::llm::{
|
||||
CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ToolCompletionRequest,
|
||||
ToolCompletionResponse,
|
||||
};
|
||||
|
||||
const AUTH_TOKEN: &str = "test-openai-token";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Mock LLM provider
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
struct MockLlmProvider;
|
||||
|
||||
#[async_trait]
|
||||
impl LlmProvider for MockLlmProvider {
|
||||
fn model_name(&self) -> &str {
|
||||
"mock-model-v1"
|
||||
}
|
||||
|
||||
fn cost_per_token(&self) -> (Decimal, Decimal) {
|
||||
(Decimal::ZERO, Decimal::ZERO)
|
||||
}
|
||||
|
||||
async fn complete(&self, req: CompletionRequest) -> Result<CompletionResponse, LlmError> {
|
||||
// Echo the last user message back
|
||||
let user_msg = req
|
||||
.messages
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|m| m.role == ironclaw::llm::Role::User)
|
||||
.map(|m| m.content.clone())
|
||||
.unwrap_or_else(|| "no user message".to_string());
|
||||
|
||||
Ok(CompletionResponse {
|
||||
content: format!("Mock response to: {}", user_msg),
|
||||
input_tokens: 10,
|
||||
output_tokens: 5,
|
||||
finish_reason: FinishReason::Stop,
|
||||
response_id: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn complete_with_tools(
|
||||
&self,
|
||||
req: ToolCompletionRequest,
|
||||
) -> Result<ToolCompletionResponse, LlmError> {
|
||||
// If tools are provided, return a tool call
|
||||
if let Some(tool) = req.tools.first() {
|
||||
Ok(ToolCompletionResponse {
|
||||
content: None,
|
||||
tool_calls: vec![ironclaw::llm::ToolCall {
|
||||
id: "call_mock_001".to_string(),
|
||||
name: tool.name.clone(),
|
||||
arguments: serde_json::json!({"test": true}),
|
||||
}],
|
||||
input_tokens: 15,
|
||||
output_tokens: 8,
|
||||
finish_reason: FinishReason::ToolUse,
|
||||
response_id: None,
|
||||
})
|
||||
} else {
|
||||
Ok(ToolCompletionResponse {
|
||||
content: Some("No tools available".to_string()),
|
||||
tool_calls: vec![],
|
||||
input_tokens: 10,
|
||||
output_tokens: 4,
|
||||
finish_reason: FinishReason::Stop,
|
||||
response_id: None,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async fn list_models(&self) -> Result<Vec<String>, LlmError> {
|
||||
Ok(vec![
|
||||
"mock-model-v1".to_string(),
|
||||
"mock-model-v2".to_string(),
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async fn start_test_server() -> (SocketAddr, Arc<GatewayState>) {
|
||||
let state = Arc::new(GatewayState {
|
||||
msg_tx: tokio::sync::RwLock::new(None),
|
||||
sse: SseManager::new(),
|
||||
workspace: None,
|
||||
session_manager: None,
|
||||
log_broadcaster: None,
|
||||
extension_manager: None,
|
||||
tool_registry: None,
|
||||
store: None,
|
||||
job_manager: None,
|
||||
prompt_queue: None,
|
||||
user_id: "test-user".to_string(),
|
||||
shutdown_tx: tokio::sync::RwLock::new(None),
|
||||
ws_tracker: Some(Arc::new(WsConnectionTracker::new())),
|
||||
llm_provider: Some(Arc::new(MockLlmProvider)),
|
||||
chat_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(30, 60),
|
||||
});
|
||||
|
||||
let addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
|
||||
let bound_addr = start_server(addr, state.clone(), AUTH_TOKEN.to_string())
|
||||
.await
|
||||
.expect("Failed to start test server");
|
||||
|
||||
(bound_addr, state)
|
||||
}
|
||||
|
||||
fn client() -> reqwest::Client {
|
||||
reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(10))
|
||||
.build()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_chat_completions_basic() {
|
||||
let (addr, _state) = start_test_server().await;
|
||||
let url = format!("http://{}/v1/chat/completions", addr);
|
||||
|
||||
let resp = client()
|
||||
.post(&url)
|
||||
.bearer_auth(AUTH_TOKEN)
|
||||
.json(&serde_json::json!({
|
||||
"model": "mock-model-v1",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Hello world"}
|
||||
]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 200);
|
||||
|
||||
let body: serde_json::Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["object"], "chat.completion");
|
||||
assert_eq!(body["model"], "mock-model-v1");
|
||||
assert_eq!(body["choices"][0]["finish_reason"], "stop");
|
||||
|
||||
let content = body["choices"][0]["message"]["content"].as_str().unwrap();
|
||||
assert!(
|
||||
content.contains("Hello world"),
|
||||
"Expected echo, got: {}",
|
||||
content
|
||||
);
|
||||
|
||||
// Check usage
|
||||
assert_eq!(body["usage"]["prompt_tokens"], 10);
|
||||
assert_eq!(body["usage"]["completion_tokens"], 5);
|
||||
assert_eq!(body["usage"]["total_tokens"], 15);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_chat_completions_with_system_message() {
|
||||
let (addr, _state) = start_test_server().await;
|
||||
let url = format!("http://{}/v1/chat/completions", addr);
|
||||
|
||||
let resp = client()
|
||||
.post(&url)
|
||||
.bearer_auth(AUTH_TOKEN)
|
||||
.json(&serde_json::json!({
|
||||
"model": "mock-model-v1",
|
||||
"messages": [
|
||||
{"role": "system", "content": "You are helpful."},
|
||||
{"role": "user", "content": "What is 2+2?"}
|
||||
],
|
||||
"temperature": 0.5,
|
||||
"max_tokens": 100
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 200);
|
||||
let body: serde_json::Value = resp.json().await.unwrap();
|
||||
let content = body["choices"][0]["message"]["content"].as_str().unwrap();
|
||||
assert!(content.contains("2+2"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_chat_completions_with_tools() {
|
||||
let (addr, _state) = start_test_server().await;
|
||||
let url = format!("http://{}/v1/chat/completions", addr);
|
||||
|
||||
let resp = client()
|
||||
.post(&url)
|
||||
.bearer_auth(AUTH_TOKEN)
|
||||
.json(&serde_json::json!({
|
||||
"model": "mock-model-v1",
|
||||
"messages": [
|
||||
{"role": "user", "content": "What's the weather?"}
|
||||
],
|
||||
"tools": [{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"description": "Get the weather",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"location": {"type": "string"}
|
||||
}
|
||||
}
|
||||
}
|
||||
}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 200);
|
||||
let body: serde_json::Value = resp.json().await.unwrap();
|
||||
|
||||
assert_eq!(body["choices"][0]["finish_reason"], "tool_calls");
|
||||
|
||||
let tool_calls = &body["choices"][0]["message"]["tool_calls"];
|
||||
assert!(tool_calls.is_array());
|
||||
assert_eq!(tool_calls[0]["id"], "call_mock_001");
|
||||
assert_eq!(tool_calls[0]["type"], "function");
|
||||
assert_eq!(tool_calls[0]["function"]["name"], "get_weather");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_chat_completions_streaming() {
|
||||
let (addr, _state) = start_test_server().await;
|
||||
let url = format!("http://{}/v1/chat/completions", addr);
|
||||
|
||||
let resp = client()
|
||||
.post(&url)
|
||||
.bearer_auth(AUTH_TOKEN)
|
||||
.json(&serde_json::json!({
|
||||
"model": "mock-model-v1",
|
||||
"messages": [
|
||||
{"role": "user", "content": "Stream test"}
|
||||
],
|
||||
"stream": true
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 200);
|
||||
|
||||
// Check simulated streaming header
|
||||
assert_eq!(
|
||||
resp.headers()
|
||||
.get("x-ironclaw-streaming")
|
||||
.and_then(|v| v.to_str().ok()),
|
||||
Some("simulated"),
|
||||
"Expected x-ironclaw-streaming: simulated header"
|
||||
);
|
||||
|
||||
let text = resp.text().await.unwrap();
|
||||
|
||||
// Should contain SSE data lines
|
||||
assert!(
|
||||
text.contains("data:"),
|
||||
"Expected SSE data lines, got: {}",
|
||||
text
|
||||
);
|
||||
// Should end with [DONE]
|
||||
assert!(
|
||||
text.contains("[DONE]"),
|
||||
"Expected [DONE] sentinel, got: {}",
|
||||
text
|
||||
);
|
||||
// Should contain the role chunk
|
||||
assert!(
|
||||
text.contains("\"role\":\"assistant\""),
|
||||
"Expected role chunk, got: {}",
|
||||
text
|
||||
);
|
||||
|
||||
// Collect all content from the chunks
|
||||
let mut full_content = String::new();
|
||||
for line in text.lines() {
|
||||
if let Some(data) = line.strip_prefix("data:") {
|
||||
let data = data.trim();
|
||||
if data == "[DONE]" {
|
||||
continue;
|
||||
}
|
||||
if let Ok(chunk) = serde_json::from_str::<serde_json::Value>(data) {
|
||||
if let Some(content) = chunk["choices"][0]["delta"]["content"].as_str() {
|
||||
full_content.push_str(content);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
full_content.contains("Stream test"),
|
||||
"Expected reassembled content to contain 'Stream test', got: '{}'",
|
||||
full_content
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_chat_completions_empty_messages() {
|
||||
let (addr, _state) = start_test_server().await;
|
||||
let url = format!("http://{}/v1/chat/completions", addr);
|
||||
|
||||
let resp = client()
|
||||
.post(&url)
|
||||
.bearer_auth(AUTH_TOKEN)
|
||||
.json(&serde_json::json!({
|
||||
"model": "mock-model-v1",
|
||||
"messages": []
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 400);
|
||||
let body: serde_json::Value = resp.json().await.unwrap();
|
||||
assert!(body["error"]["message"].as_str().unwrap().contains("empty"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_chat_completions_model_mismatch() {
|
||||
let (addr, _state) = start_test_server().await;
|
||||
let url = format!("http://{}/v1/chat/completions", addr);
|
||||
|
||||
let resp = client()
|
||||
.post(&url)
|
||||
.bearer_auth(AUTH_TOKEN)
|
||||
.json(&serde_json::json!({
|
||||
"model": "gpt-4",
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 404);
|
||||
let body: serde_json::Value = resp.json().await.unwrap();
|
||||
assert_eq!(body["error"]["code"], "model_not_found");
|
||||
assert!(
|
||||
body["error"]["message"]
|
||||
.as_str()
|
||||
.unwrap()
|
||||
.contains("mock-model-v1")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_chat_completions_no_auth() {
|
||||
let (addr, _state) = start_test_server().await;
|
||||
let url = format!("http://{}/v1/chat/completions", addr);
|
||||
|
||||
let resp = client()
|
||||
.post(&url)
|
||||
// No auth header
|
||||
.json(&serde_json::json!({
|
||||
"model": "mock-model-v1",
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 401);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_models_endpoint() {
|
||||
let (addr, _state) = start_test_server().await;
|
||||
let url = format!("http://{}/v1/models", addr);
|
||||
|
||||
let resp = client()
|
||||
.get(&url)
|
||||
.bearer_auth(AUTH_TOKEN)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 200);
|
||||
let body: serde_json::Value = resp.json().await.unwrap();
|
||||
|
||||
assert_eq!(body["object"], "list");
|
||||
let data = body["data"].as_array().unwrap();
|
||||
assert_eq!(data.len(), 2);
|
||||
assert_eq!(data[0]["id"], "mock-model-v1");
|
||||
assert_eq!(data[1]["id"], "mock-model-v2");
|
||||
assert_eq!(data[0]["object"], "model");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_models_no_auth() {
|
||||
let (addr, _state) = start_test_server().await;
|
||||
let url = format!("http://{}/v1/models", addr);
|
||||
|
||||
let resp = client().get(&url).send().await.unwrap();
|
||||
assert_eq!(resp.status(), 401);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_no_llm_provider_returns_503() {
|
||||
// Create state WITHOUT llm_provider
|
||||
let state = Arc::new(GatewayState {
|
||||
msg_tx: tokio::sync::RwLock::new(None),
|
||||
sse: SseManager::new(),
|
||||
workspace: None,
|
||||
session_manager: None,
|
||||
log_broadcaster: None,
|
||||
extension_manager: None,
|
||||
tool_registry: None,
|
||||
store: None,
|
||||
job_manager: None,
|
||||
prompt_queue: None,
|
||||
user_id: "test-user".to_string(),
|
||||
shutdown_tx: tokio::sync::RwLock::new(None),
|
||||
ws_tracker: Some(Arc::new(WsConnectionTracker::new())),
|
||||
llm_provider: None, // No LLM!
|
||||
chat_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(30, 60),
|
||||
});
|
||||
|
||||
let addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
|
||||
let bound_addr = start_server(addr, state, AUTH_TOKEN.to_string())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let url = format!("http://{}/v1/chat/completions", bound_addr);
|
||||
let resp = client()
|
||||
.post(&url)
|
||||
.bearer_auth(AUTH_TOKEN)
|
||||
.json(&serde_json::json!({
|
||||
"model": "mock-model-v1",
|
||||
"messages": [{"role": "user", "content": "Hi"}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 503);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_chat_completions_body_too_large() {
|
||||
let (addr, _state) = start_test_server().await;
|
||||
let url = format!("http://{}/v1/chat/completions", addr);
|
||||
|
||||
// Build a payload over 1 MB (the gateway's DefaultBodyLimit)
|
||||
let big_content = "x".repeat(2 * 1024 * 1024);
|
||||
let resp = client()
|
||||
.post(&url)
|
||||
.bearer_auth(AUTH_TOKEN)
|
||||
.json(&serde_json::json!({
|
||||
"model": "mock-model-v1",
|
||||
"messages": [{"role": "user", "content": big_content}]
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(resp.status(), 413);
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
#![cfg(feature = "postgres")]
|
||||
//! Integration tests for the workspace module.
|
||||
//!
|
||||
//! Requires a running PostgreSQL with pgvector extension.
|
||||
@@ -20,6 +21,18 @@ fn get_pool() -> deadpool_postgres::Pool {
|
||||
.expect("Failed to create pool")
|
||||
}
|
||||
|
||||
/// Try to get a connection, returning None if Postgres is unreachable.
|
||||
/// Tests call this to skip gracefully in CI where no database is available.
|
||||
async fn try_connect(pool: &deadpool_postgres::Pool) -> Option<()> {
|
||||
match pool.get().await {
|
||||
Ok(_) => Some(()),
|
||||
Err(e) => {
|
||||
eprintln!("skipping: database unavailable ({e})");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn cleanup_user(pool: &deadpool_postgres::Pool, user_id: &str) {
|
||||
let conn = pool.get().await.expect("Failed to get connection");
|
||||
conn.execute(
|
||||
@@ -33,6 +46,9 @@ async fn cleanup_user(pool: &deadpool_postgres::Pool, user_id: &str) {
|
||||
#[tokio::test]
|
||||
async fn test_workspace_write_and_read() {
|
||||
let pool = get_pool();
|
||||
if try_connect(&pool).await.is_none() {
|
||||
return;
|
||||
}
|
||||
let user_id = "test_write_read";
|
||||
cleanup_user(&pool, user_id).await;
|
||||
|
||||
@@ -58,6 +74,9 @@ async fn test_workspace_write_and_read() {
|
||||
#[tokio::test]
|
||||
async fn test_workspace_append() {
|
||||
let pool = get_pool();
|
||||
if try_connect(&pool).await.is_none() {
|
||||
return;
|
||||
}
|
||||
let user_id = "test_append";
|
||||
cleanup_user(&pool, user_id).await;
|
||||
|
||||
@@ -85,6 +104,9 @@ async fn test_workspace_append() {
|
||||
#[tokio::test]
|
||||
async fn test_workspace_nested_paths() {
|
||||
let pool = get_pool();
|
||||
if try_connect(&pool).await.is_none() {
|
||||
return;
|
||||
}
|
||||
let user_id = "test_nested";
|
||||
cleanup_user(&pool, user_id).await;
|
||||
|
||||
@@ -130,6 +152,9 @@ async fn test_workspace_nested_paths() {
|
||||
#[tokio::test]
|
||||
async fn test_workspace_delete() {
|
||||
let pool = get_pool();
|
||||
if try_connect(&pool).await.is_none() {
|
||||
return;
|
||||
}
|
||||
let user_id = "test_delete";
|
||||
cleanup_user(&pool, user_id).await;
|
||||
|
||||
@@ -154,6 +179,9 @@ async fn test_workspace_delete() {
|
||||
#[tokio::test]
|
||||
async fn test_workspace_memory_operations() {
|
||||
let pool = get_pool();
|
||||
if try_connect(&pool).await.is_none() {
|
||||
return;
|
||||
}
|
||||
let user_id = "test_memory_ops";
|
||||
cleanup_user(&pool, user_id).await;
|
||||
|
||||
@@ -182,6 +210,9 @@ async fn test_workspace_memory_operations() {
|
||||
#[tokio::test]
|
||||
async fn test_workspace_daily_log() {
|
||||
let pool = get_pool();
|
||||
if try_connect(&pool).await.is_none() {
|
||||
return;
|
||||
}
|
||||
let user_id = "test_daily_log";
|
||||
cleanup_user(&pool, user_id).await;
|
||||
|
||||
@@ -208,6 +239,9 @@ async fn test_workspace_daily_log() {
|
||||
#[tokio::test]
|
||||
async fn test_workspace_fts_search() {
|
||||
let pool = get_pool();
|
||||
if try_connect(&pool).await.is_none() {
|
||||
return;
|
||||
}
|
||||
let user_id = "test_fts_search";
|
||||
cleanup_user(&pool, user_id).await;
|
||||
|
||||
@@ -266,6 +300,9 @@ async fn test_workspace_fts_search() {
|
||||
#[tokio::test]
|
||||
async fn test_workspace_hybrid_search_with_mock_embeddings() {
|
||||
let pool = get_pool();
|
||||
if try_connect(&pool).await.is_none() {
|
||||
return;
|
||||
}
|
||||
let user_id = "test_hybrid_search";
|
||||
cleanup_user(&pool, user_id).await;
|
||||
|
||||
@@ -305,6 +342,9 @@ async fn test_workspace_hybrid_search_with_mock_embeddings() {
|
||||
#[tokio::test]
|
||||
async fn test_workspace_list_all() {
|
||||
let pool = get_pool();
|
||||
if try_connect(&pool).await.is_none() {
|
||||
return;
|
||||
}
|
||||
let user_id = "test_list_all";
|
||||
cleanup_user(&pool, user_id).await;
|
||||
|
||||
@@ -330,6 +370,9 @@ async fn test_workspace_list_all() {
|
||||
#[tokio::test]
|
||||
async fn test_workspace_system_prompt() {
|
||||
let pool = get_pool();
|
||||
if try_connect(&pool).await.is_none() {
|
||||
return;
|
||||
}
|
||||
let user_id = "test_system_prompt";
|
||||
cleanup_user(&pool, user_id).await;
|
||||
|
||||
|
||||
@@ -51,6 +51,8 @@ async fn start_test_server() -> (
|
||||
user_id: "test-user".to_string(),
|
||||
shutdown_tx: tokio::sync::RwLock::new(None),
|
||||
ws_tracker: Some(Arc::new(WsConnectionTracker::new())),
|
||||
llm_provider: None,
|
||||
chat_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(30, 60),
|
||||
});
|
||||
|
||||
let addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
|
||||
@@ -66,7 +68,12 @@ async fn connect_ws(
|
||||
addr: SocketAddr,
|
||||
) -> tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>> {
|
||||
let url = format!("ws://{}/api/chat/ws?token={}", addr, AUTH_TOKEN);
|
||||
let request = url.into_client_request().unwrap();
|
||||
let mut request = url.into_client_request().unwrap();
|
||||
// Server requires an Origin header from localhost to prevent cross-site WS hijacking.
|
||||
request.headers_mut().insert(
|
||||
"Origin",
|
||||
format!("http://127.0.0.1:{}", addr.port()).parse().unwrap(),
|
||||
);
|
||||
let (stream, _response) = tokio_tungstenite::connect_async(request)
|
||||
.await
|
||||
.expect("Failed to connect WebSocket");
|
||||
|
||||
Reference in New Issue
Block a user