From e843c18141ad33deb876e6106818703a105a6b85 Mon Sep 17 00:00:00 2001 From: Zaki Manian Date: Fri, 13 Feb 2026 18:05:05 -0800 Subject: [PATCH] feat: add libSQL/Turso embedded database backend (#47) * feat: add libSQL/Turso database backend with full feature parity Introduce a Database trait abstraction (~60 async methods) enabling compile-time backend selection between PostgreSQL and libSQL/Turso. Convert all modules from concrete Store to Arc, add LibSqlSecretsStore and LibSqlWasmToolStore implementations, wire libsql stores throughout CLI and main entry points, and make the setup wizard backend-agnostic. Key changes: - src/db/: Database trait, PostgresDatabase adapter, LibSqlBackend with native SQLite-dialect SQL, and idempotent migration system - src/secrets/store.rs: LibSqlSecretsStore (all 8 trait methods) - src/tools/wasm/storage.rs: LibSqlWasmToolStore (all 7 trait methods) - src/main.rs, cli/tool.rs, cli/mcp.rs: backend-conditional wiring - src/setup/channels.rs: SecretsContext uses Arc - Feature-gate postgres-only tests and examples Co-Authored-By: Claude Opus 4.6 * feat: enable onboarding wizard for libSQL builds Refactor the setup wizard to work with both postgres and libsql feature flags. Previously the wizard was gated behind #[cfg(feature = "postgres")] only, so libsql-only builds would print an error on `ironclaw onboard`. - Add libsql fields to Settings (database_backend, libsql_path, libsql_url) - Split wizard database/migration/secrets methods into feature-gated variants - Add step_database_libsql() with local path and Turso remote replica prompts - Update setup/mod.rs and main.rs feature gates to any(postgres, libsql) - Extend check_onboard_needed() to detect libsql database presence Co-Authored-By: Claude Opus 4.6 * fix: address PR review feedback for libSQL backend - P0: Switch libsql_backend to connection-per-operation pattern to fix shared Connection concurrency issue across tokio tasks - P0: Wrap secrets store INSERT+SELECT in transaction to fix TOCTOU race - P0: Document encryption-at-rest limitations and json_patch divergence - P1: Fix get_opt_text removing .filter(|s| !s.is_empty()) that conflated empty strings with NULL - P1: Replace datetime('now') with fmt_ts(&Utc::now()) for consistent RFC 3339 timestamps across all queries - P2: Use explicit _rowid column in FTS5 triggers and joins for stability across VACUUM operations - P2: Add tracing::warn when embedding provided but vector search disabled in hybrid_search - Extract shared connect_from_config() helper to deduplicate DB connection logic across main.rs, cli/config.rs, and cli/mcp.rs Co-Authored-By: Claude Opus 4.6 * fix: add missing JobContext fields and resolve fmt/clippy warnings Add total_tokens_used and max_tokens fields to JobContext in libsql_backend.rs, apply cargo fmt, and fix clippy warnings. Co-Authored-By: Claude Opus 4.6 * fix: review fixes for libSQL backend (shared connections, panics, indexes) - Replace .expect() with proper error propagation in 3 call sites - Share Arc between backend and stores instead of single Connection - Add connect-per-operation pattern to LibSqlSecretsStore and LibSqlWasmToolStore - Wrap store() INSERT + SELECT-back in a transaction - Add ~22 missing indexes for parity with PostgreSQL schema - Add 18 leak_detection_patterns seed rows matching PostgreSQL V2 migration - Fix super:: import to use crate:: style - Gate mask_password_in_url behind #[cfg(feature = "postgres")] - Rewrite secrets store init with or_else chain for runtime backend selection Co-Authored-By: Claude Opus 4.6 * fix: Resolve clippy lints (collapsible_if, too_many_arguments) Collapse nested if blocks into let_chains to satisfy clippy's collapsible_if lint (CI uses -D warnings). Suppress too_many_arguments on libsql_row_to_tool_at since refactoring the positional index pattern would be a larger change. Co-Authored-By: Claude Opus 4.6 --------- Co-authored-by: Claude Opus 4.6 Co-authored-by: Illia Polosukhin --- CLAUDE.md | 100 +- Cargo.lock | 694 ++++++++- Cargo.toml | 32 +- src/agent/agent_loop.rs | 6 +- src/agent/routine_engine.rs | 8 +- src/agent/scheduler.rs | 6 +- src/agent/self_repair.rs | 6 +- src/agent/worker.rs | 8 +- src/bootstrap.rs | 2 +- src/channels/web/mod.rs | 4 +- src/channels/web/server.rs | 4 +- src/cli/config.rs | 41 +- src/cli/mcp.rs | 117 +- src/cli/memory.rs | 27 +- src/cli/mod.rs | 5 +- src/cli/status.rs | 7 + src/cli/tool.rs | 76 +- src/config.rs | 87 +- src/db/libsql_backend.rs | 2609 ++++++++++++++++++++++++++++++++ src/db/libsql_migrations.rs | 549 +++++++ src/db/mod.rs | 538 +++++++ src/db/postgres.rs | 627 ++++++++ src/error.rs | 7 + src/extensions/manager.rs | 12 +- src/history/mod.rs | 6 +- src/history/store.rs | 23 + src/lib.rs | 1 + src/llm/session.rs | 4 +- src/main.rs | 220 ++- src/orchestrator/api.rs | 4 +- src/secrets/mod.rs | 6 +- src/secrets/store.rs | 331 ++++ src/settings.rs | 12 + src/setup/channels.rs | 19 +- src/setup/mod.rs | 2 + src/setup/wizard.rs | 334 +++- src/tools/builtin/job.rs | 7 +- src/tools/builtin/memory.rs | 2 +- src/tools/builtin/routine.rs | 22 +- src/tools/mcp/config.rs | 8 +- src/tools/registry.rs | 6 +- src/tools/wasm/mod.rs | 9 +- src/tools/wasm/storage.rs | 458 ++++++ src/workspace/mod.rs | 247 ++- tests/workspace_integration.rs | 1 + 45 files changed, 6973 insertions(+), 321 deletions(-) create mode 100644 src/db/libsql_backend.rs create mode 100644 src/db/libsql_migrations.rs create mode 100644 src/db/mod.rs create mode 100644 src/db/postgres.rs diff --git a/CLAUDE.md b/CLAUDE.md index 3d3bd4a3..aeaf1dd2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -151,6 +151,12 @@ src/ │ ├── rate_limiter.rs # Per-tool rate limiting │ └── storage.rs # Linear memory persistence │ +├── db/ # Database abstraction layer +│ ├── mod.rs # Database trait (~60 async methods) +│ ├── postgres.rs # PostgreSQL backend (delegates to Store + Repository) +│ ├── libsql_backend.rs # libSQL/Turso backend (embedded SQLite) +│ └── libsql_migrations.rs # SQLite-dialect schema (idempotent) +│ ├── workspace/ # Persistent memory system (OpenClaw-inspired) │ ├── mod.rs # Workspace struct, memory operations │ ├── document.rs # MemoryDocument, MemoryChunk, WorkspaceEntry @@ -201,6 +207,7 @@ When designing new features or systems, always prefer generic/extensible archite - Use `RwLock` for concurrent read/write access ### Traits for Extensibility +- `Database` - Add new database backends (must implement all ~60 methods) - `Channel` - Add new input sources - `Tool` - Add new capabilities - `LlmProvider` - Add new LLM backends @@ -248,7 +255,12 @@ Pending -> InProgress -> Completed -> Submitted -> Accepted Environment variables (see `.env.example`): ```bash +# Database backend (default: postgres) +DATABASE_BACKEND=postgres # or "libsql" / "turso" DATABASE_URL=postgres://user:pass@localhost/ironclaw +LIBSQL_PATH=~/.ironclaw/ironclaw.db # libSQL local path (default) +# LIBSQL_URL=libsql://xxx.turso.io # Turso cloud (optional) +# LIBSQL_AUTH_TOKEN=xxx # Required with LIBSQL_URL # NEAR AI (required) NEARAI_SESSION_TOKEN=sess_... @@ -308,7 +320,51 @@ Session tokens have the format `sess_xxx` (37 characters). They are authenticate ## Database -Single migration in `migrations/V1__initial.sql`. Tables: +IronClaw supports two database backends, selected at compile time via Cargo feature flags and at runtime via the `DATABASE_BACKEND` environment variable. + +**IMPORTANT: All new features that touch persistence MUST support both backends.** Implement the operation as a method on the `Database` trait in `src/db/mod.rs`, then add the implementation in both `src/db/postgres.rs` (delegate to Store/Repository) and `src/db/libsql_backend.rs` (native SQL). + +### Backends + +| Backend | Feature Flag | Default | Use Case | +|---------|-------------|---------|----------| +| PostgreSQL | `postgres` (default) | Yes | Production, existing deployments | +| libSQL/Turso | `libsql` | No | Zero-dependency local mode, edge, Turso cloud | + +```bash +# Build with PostgreSQL only (default) +cargo build + +# Build with libSQL only +cargo build --no-default-features --features libsql + +# Build with both backends available +cargo build --features "postgres,libsql" +``` + +### Database Trait + +The `Database` trait (`src/db/mod.rs`) defines ~60 async methods covering all persistence: +- Conversations, messages, metadata +- Jobs, actions, LLM calls, estimation snapshots +- Sandbox jobs, job events +- Routines, routine runs +- Tool failures, settings +- Workspace: documents, chunks, hybrid search + +Both backends implement this trait. PostgreSQL delegates to the existing `Store` + `Repository`. libSQL implements native SQLite-dialect SQL. + +### Schema + +**PostgreSQL:** `migrations/V1__initial.sql` (351 lines). Uses pgvector for embeddings, tsvector for FTS, PL/pgSQL functions. Managed by `refinery`. + +**libSQL:** `src/db/libsql_migrations.rs` (consolidated schema, ~480 lines). Translates PG types: +- `UUID` -> `TEXT`, `TIMESTAMPTZ` -> `TEXT` (ISO-8601), `JSONB` -> `TEXT` +- `VECTOR(1536)` -> `F32_BLOB(1536)` with `libsql_vector_idx` +- `tsvector`/`ts_rank_cd` -> FTS5 virtual table with sync triggers +- PL/pgSQL functions -> SQLite triggers + +**Tables (both backends):** **Core:** - `conversations` - Multi-channel conversation tracking @@ -320,12 +376,41 @@ Single migration in `migrations/V1__initial.sql`. Tables: **Workspace/Memory:** - `memory_documents` - Flexible path-based files (e.g., "context/vision.md", "daily/2024-01-15.md") -- `memory_chunks` - Chunked content with FTS (tsvector) and vector (pgvector) indexes +- `memory_chunks` - Chunked content with FTS and vector indexes - `heartbeat_state` - Periodic execution tracking -Requires pgvector extension: `CREATE EXTENSION IF NOT EXISTS vector;` +**Other:** +- `routines`, `routine_runs` - Scheduled/reactive execution +- `settings` - Per-user key-value settings +- `tool_failures` - Self-repair tracking +- `secrets`, `wasm_tools`, `tool_capabilities` - Extension infrastructure -Run migrations: `refinery migrate -c refinery.toml` +### Configuration + +```bash +# Backend selection (default: postgres) +DATABASE_BACKEND=libsql + +# PostgreSQL +DATABASE_URL=postgres://user:pass@localhost/ironclaw + +# libSQL (embedded) +LIBSQL_PATH=~/.ironclaw/ironclaw.db # Default path + +# libSQL (Turso cloud sync) +LIBSQL_URL=libsql://your-db.turso.io +LIBSQL_AUTH_TOKEN=your-token # Required when LIBSQL_URL is set +``` + +### Current Limitations (libSQL backend) + +- **Workspace/memory system** not yet wired through Database trait (requires Store migration) +- **Secrets store** not yet available (still requires PostgresSecretsStore) +- **Hybrid search** uses FTS5 only (vector search via libsql_vector_idx not yet implemented) +- **Settings reload from DB** skipped (Config::from_db requires Store) +- No incremental migration versioning (schema is CREATE IF NOT EXISTS, no ALTER TABLE support yet) +- **No encryption at rest** -- The local SQLite database file stores conversation content, job data, workspace memory, and other application data in plaintext. Only secrets (API tokens, credentials) are encrypted via AES-256-GCM before storage. Users handling sensitive data should use full-disk encryption (FileVault, LUKS, BitLocker) or consider the PostgreSQL backend with TDE/encrypted storage. +- **JSON merge patch vs path-targeted update** -- The libSQL backend uses RFC 7396 JSON Merge Patch (`json_patch`) for metadata updates, while PostgreSQL uses path-targeted `jsonb_set`. Merge patch replaces top-level keys entirely, which may drop nested keys not present in the patch. Callers should avoid relying on partial nested object updates in metadata fields. ## Safety Layer @@ -387,6 +472,7 @@ Key test patterns: - ✅ **Claude Code mode** - Delegate jobs to Claude CLI inside containers - ✅ **Routines system** - Cron, event, webhook, and manual triggers with guardrails - ✅ **Extension management** - Install, auth, activate MCP/WASM extensions via CLI and web UI +- ✅ **libSQL/Turso backend** - Database trait abstraction (`src/db/`), feature-gated dual backend support (postgres/libsql), embedded SQLite for zero-dependency local mode ## Adding a New Tool @@ -625,7 +711,7 @@ Four tools for LLM use: ### Hybrid Search (RRF) -Combines full-text search (PostgreSQL `ts_rank_cd`) and vector similarity (pgvector cosine) using Reciprocal Rank Fusion: +Combines full-text search and vector similarity using Reciprocal Rank Fusion: ``` score(d) = Σ 1/(k + rank(d)) for each method where d appears @@ -633,6 +719,10 @@ score(d) = Σ 1/(k + rank(d)) for each method where d appears Default k=60. Results from both methods are combined, with documents appearing in both getting boosted scores. +**Backend differences:** +- **PostgreSQL:** `ts_rank_cd` for FTS, pgvector cosine distance for vectors, full RRF +- **libSQL:** FTS5 for keyword search only (vector search via `libsql_vector_idx` not yet wired) + ### Heartbeat System Proactive periodic execution (default: 30 minutes): diff --git a/Cargo.lock b/Cargo.lock index be2dcb82..6d64e505 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -66,7 +66,7 @@ dependencies = [ "cfg-if", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.8.37", ] [[package]] @@ -364,24 +364,52 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "axum" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b829e4e32b91e643de6eafe82b1d90675f5874230191a4ffbc1b336dec4d6bf" +dependencies = [ + "async-trait", + "axum-core 0.3.4", + "bitflags 1.3.2", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "itoa", + "matchit 0.7.3", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 0.1.2", + "tower 0.4.13", + "tower-layer", + "tower-service", +] + [[package]] name = "axum" version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b52af3cb4058c895d37317bb27508dccc8e5f2d39454016b297bf4a400597b8" dependencies = [ - "axum-core", + "axum-core 0.5.6", "base64 0.22.1", "bytes", "form_urlencoded", "futures-util", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", "itoa", - "matchit", + "matchit 0.8.4", "memchr", "mime", "percent-encoding", @@ -391,15 +419,32 @@ dependencies = [ "serde_path_to_error", "serde_urlencoded", "sha1", - "sync_wrapper", + "sync_wrapper 1.0.2", "tokio", "tokio-tungstenite 0.28.0", - "tower", + "tower 0.5.3", "tower-layer", "tower-service", "tracing", ] +[[package]] +name = "axum-core" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "mime", + "rustversion", + "tower-layer", + "tower-service", +] + [[package]] name = "axum-core" version = "0.5.6" @@ -408,12 +453,12 @@ checksum = "08c78f31d7b1291f7ee735c1c6780ccde7785daae9a9206026862dab7d8792d1" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", "mime", "pin-project-lite", - "sync_wrapper", + "sync_wrapper 1.0.2", "tower-layer", "tower-service", "tracing", @@ -431,6 +476,38 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bindgen" +version = "0.66.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b84e06fc203107bfbad243f4aba2af864eb7db3b1cf46ea0a023b0b433d2a7" +dependencies = [ + "bitflags 2.10.0", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.114", + "which", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -513,9 +590,9 @@ dependencies = [ "futures-util", "hex", "home", - "http", + "http 1.4.0", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-named-pipe", "hyper-rustls", "hyper-util", @@ -615,6 +692,9 @@ name = "bytes" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +dependencies = [ + "serde", +] [[package]] name = "cap-fs-ext" @@ -715,6 +795,15 @@ dependencies = [ "shlex", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -751,6 +840,17 @@ dependencies = [ "inout", ] +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "clap" version = "4.5.56" @@ -929,7 +1029,7 @@ dependencies = [ "hashbrown 0.14.5", "log", "regalloc2", - "rustc-hash", + "rustc-hash 2.1.1", "serde", "smallvec", "target-lexicon", @@ -1540,6 +1640,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.3.0" @@ -1831,6 +1937,25 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "h2" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0beca50380b1fc32983fc1cb4587bfa4bb9e78fc259aad4a0032d2080309222d" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 0.2.12", + "indexmap 2.13.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "h2" version = "0.4.13" @@ -1842,7 +1967,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.4.0", "indexmap 2.13.0", "slab", "tokio", @@ -1866,6 +1991,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash 0.8.12", + "allocator-api2", "serde", ] @@ -1885,6 +2011,15 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "heck" version = "0.5.0" @@ -1930,6 +2065,17 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.4.0" @@ -1940,6 +2086,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" +dependencies = [ + "bytes", + "http 0.2.12", + "pin-project-lite", +] + [[package]] name = "http-body" version = "1.0.1" @@ -1947,7 +2104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.4.0", ] [[package]] @@ -1958,11 +2115,17 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "pin-project-lite", ] +[[package]] +name = "http-range-header" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" + [[package]] name = "httparse" version = "1.10.1" @@ -1975,6 +2138,30 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hyper" +version = "0.14.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.5.10", + "tokio", + "tower-service", + "tracing", + "want", +] + [[package]] name = "hyper" version = "1.8.1" @@ -1985,9 +2172,9 @@ dependencies = [ "bytes", "futures-channel", "futures-core", - "h2", - "http", - "http-body", + "h2 0.4.13", + "http 1.4.0", + "http-body 1.0.1", "httparse", "httpdate", "itoa", @@ -2005,7 +2192,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" dependencies = [ "hex", - "hyper", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -2019,8 +2206,8 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http", - "hyper", + "http 1.4.0", + "hyper 1.8.1", "hyper-util", "rustls", "rustls-pki-types", @@ -2030,6 +2217,18 @@ dependencies = [ "webpki-roots", ] +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper 0.14.32", + "pin-project-lite", + "tokio", + "tokio-io-timeout", +] + [[package]] name = "hyper-tls" version = "0.6.0" @@ -2038,7 +2237,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", "native-tls", "tokio", @@ -2056,14 +2255,14 @@ dependencies = [ "bytes", "futures-channel", "futures-util", - "http", - "http-body", - "hyper", + "http 1.4.0", + "http-body 1.0.1", + "hyper 1.8.1", "ipnet", "libc", "percent-encoding", "pin-project-lite", - "socket2", + "socket2 0.6.2", "system-configuration", "tokio", "tower-service", @@ -2079,7 +2278,7 @@ checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" dependencies = [ "hex", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", "pin-project-lite", "tokio", @@ -2297,7 +2496,7 @@ dependencies = [ "aho-corasick", "anyhow", "async-trait", - "axum", + "axum 0.8.8", "base64 0.22.1", "blake3", "bollard", @@ -2313,8 +2512,9 @@ dependencies = [ "futures", "hkdf", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-util", + "libsql", "mime_guess", "open", "pgvector", @@ -2344,8 +2544,8 @@ dependencies = [ "tokio-stream", "tokio-test", "tokio-tungstenite 0.26.2", - "tower", - "tower-http", + "tower 0.5.3", + "tower-http 0.6.8", "tracing", "tracing-subscriber", "urlencoding", @@ -2465,6 +2665,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "leb128" version = "0.2.5" @@ -2483,6 +2689,16 @@ version = "0.2.180" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -2500,6 +2716,121 @@ dependencies = [ "redox_syscall 0.7.0", ] +[[package]] +name = "libsql" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe18646e4ef8db446bc3e3f5fb96131483203bc5f4998ff149f79a067530c01c" +dependencies = [ + "anyhow", + "async-stream", + "async-trait", + "bincode", + "bitflags 2.10.0", + "bytes", + "fallible-iterator 0.3.0", + "futures", + "http 0.2.12", + "hyper 0.14.32", + "libsql-sqlite3-parser", + "libsql-sys", + "libsql_replication", + "parking_lot", + "serde", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tonic", + "tonic-web", + "tower 0.4.13", + "tower-http 0.4.4", + "tracing", + "uuid", + "zerocopy 0.7.35", +] + +[[package]] +name = "libsql-ffi" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f2a50a585a1184a43621a9133b7702ba5cb7a87ca5e704056b19d8005de6faf" +dependencies = [ + "bindgen", + "cc", +] + +[[package]] +name = "libsql-rusqlite" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae65c66088dcd309abbd5617ae046abac2a2ee0a7fdada5127353bd68e0a27ea" +dependencies = [ + "bitflags 2.10.0", + "fallible-iterator 0.2.0", + "fallible-streaming-iterator", + "hashlink", + "libsql-ffi", + "smallvec", +] + +[[package]] +name = "libsql-sqlite3-parser" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15a90128c708356af8f7d767c9ac2946692c9112b4f74f07b99a01a60680e413" +dependencies = [ + "bitflags 2.10.0", + "cc", + "fallible-iterator 0.3.0", + "indexmap 2.13.0", + "log", + "memchr", + "phf 0.11.3", + "phf_codegen", + "phf_shared 0.11.3", + "uncased", +] + +[[package]] +name = "libsql-sys" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c05b61c226781d6f5e26e3e7364617f19c0c1d5332035802e9229d6024cec05" +dependencies = [ + "bytes", + "libsql-ffi", + "libsql-rusqlite", + "once_cell", + "tracing", + "zerocopy 0.7.35", +] + +[[package]] +name = "libsql_replication" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf40c4c2c01462da758272976de0a23d19b4e9c714db08efecf262d896655b5" +dependencies = [ + "aes", + "async-stream", + "async-trait", + "bytes", + "cbc", + "libsql-rusqlite", + "libsql-sys", + "parking_lot", + "prost", + "serde", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "tracing", + "uuid", + "zerocopy 0.7.35", +] + [[package]] name = "linux-raw-sys" version = "0.4.15" @@ -2563,6 +2894,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "matchit" version = "0.8.4" @@ -3011,6 +3348,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3027,16 +3370,55 @@ dependencies = [ "postgres-types", ] +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared 0.11.3", +] + [[package]] name = "phf" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ - "phf_shared", + "phf_shared 0.13.1", "serde", ] +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared 0.11.3", + "rand 0.8.5", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", + "uncased", +] + [[package]] name = "phf_shared" version = "0.13.1" @@ -3187,7 +3569,7 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" dependencies = [ - "zerocopy", + "zerocopy 0.8.37", ] [[package]] @@ -3200,6 +3582,16 @@ dependencies = [ "yansi", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.114", +] + [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -3218,6 +3610,29 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "psm" version = "0.1.29" @@ -3270,9 +3685,9 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.1", "rustls", - "socket2", + "socket2 0.6.2", "thiserror 2.0.18", "tokio", "tracing", @@ -3290,7 +3705,7 @@ dependencies = [ "lru-slab", "rand 0.9.2", "ring", - "rustc-hash", + "rustc-hash 2.1.1", "rustls", "rustls-pki-types", "slab", @@ -3309,7 +3724,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2", + "socket2 0.6.2", "tracing", "windows-sys 0.60.2", ] @@ -3548,7 +3963,7 @@ dependencies = [ "bumpalo", "hashbrown 0.15.5", "log", - "rustc-hash", + "rustc-hash 2.1.1", "smallvec", ] @@ -3601,11 +4016,11 @@ dependencies = [ "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.4.13", + "http 1.4.0", + "http-body 1.0.1", "http-body-util", - "hyper", + "hyper 1.8.1", "hyper-rustls", "hyper-tls", "hyper-util", @@ -3622,13 +4037,13 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 1.0.2", "tokio", "tokio-native-tls", "tokio-rustls", "tokio-util", - "tower", - "tower-http", + "tower 0.5.3", + "tower-http 0.6.8", "tower-service", "url", "wasm-bindgen", @@ -3653,7 +4068,7 @@ dependencies = [ "futures", "futures-timer", "glob", - "http", + "http 1.4.0", "mime", "mime_guess", "nanoid", @@ -3746,6 +4161,12 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.1" @@ -4273,6 +4694,16 @@ dependencies = [ "serde", ] +[[package]] +name = "socket2" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "socket2" version = "0.6.2" @@ -4375,6 +4806,12 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -4637,12 +5074,22 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.6.2", "tokio-macros", "tracing", "windows-sys 0.61.2", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bd86198d9ee903fedd2f9a2e72014287c0d9167e4ae43b5853007205dda1b76" +dependencies = [ + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-macros" version = "2.6.0" @@ -4679,12 +5126,12 @@ dependencies = [ "log", "parking_lot", "percent-encoding", - "phf", + "phf 0.13.1", "pin-project-lite", "postgres-protocol", "postgres-types", "rand 0.9.2", - "socket2", + "socket2 0.6.2", "tokio", "tokio-util", "whoami", @@ -4846,6 +5293,73 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" +[[package]] +name = "tonic" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" +dependencies = [ + "async-stream", + "async-trait", + "axum 0.6.20", + "base64 0.21.7", + "bytes", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "tokio", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-web" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc3b0e1cedbf19fdfb78ef3d672cb9928e0a91a9cb4629cc0c916e8cff8aaaa1" +dependencies = [ + "base64 0.21.7", + "bytes", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.32", + "pin-project", + "tokio-stream", + "tonic", + "tower-http 0.4.4", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower" version = "0.5.3" @@ -4855,13 +5369,33 @@ dependencies = [ "futures-core", "futures-util", "pin-project-lite", - "sync_wrapper", + "sync_wrapper 1.0.2", "tokio", "tower-layer", "tower-service", "tracing", ] +[[package]] +name = "tower-http" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" +dependencies = [ + "bitflags 2.10.0", + "bytes", + "futures-core", + "futures-util", + "http 0.2.12", + "http-body 0.4.6", + "http-range-header", + "pin-project-lite", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tower-http" version = "0.6.8" @@ -4871,11 +5405,11 @@ dependencies = [ "bitflags 2.10.0", "bytes", "futures-util", - "http", - "http-body", + "http 1.4.0", + "http-body 1.0.1", "iri-string", "pin-project-lite", - "tower", + "tower 0.5.3", "tower-layer", "tower-service", "tracing", @@ -4994,7 +5528,7 @@ checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" dependencies = [ "bytes", "data-encoding", - "http", + "http 1.4.0", "httparse", "log", "rand 0.9.2", @@ -5011,7 +5545,7 @@ checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" dependencies = [ "bytes", "data-encoding", - "http", + "http 1.4.0", "httparse", "log", "rand 0.9.2", @@ -5037,6 +5571,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "uncased" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b88fcfe09e89d3866a5c11019378088af2d24c3fbd4f0543f96b479ec90697" +dependencies = [ + "version_check", +] + [[package]] name = "unicase" version = "2.9.0" @@ -5703,6 +6246,18 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "whoami" version = "2.1.0" @@ -6287,13 +6842,34 @@ dependencies = [ "zvariant", ] +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive 0.7.35", +] + [[package]] name = "zerocopy" version = "0.8.37" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7456cf00f0685ad319c5b1693f291a650eaf345e941d082fc4e03df8a03996ac" dependencies = [ - "zerocopy-derive", + "zerocopy-derive 0.8.37", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 8ae00a3d..6a11219a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,11 +28,14 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus serde = { version = "1", features = ["derive"] } serde_json = "1" -# Database -deadpool-postgres = "0.14" -tokio-postgres = { version = "0.7", features = ["with-uuid-1", "with-chrono-0_4", "with-serde_json-1"] } -postgres-types = { version = "0.2", features = ["with-serde_json-1"] } -refinery = { version = "0.8", features = ["tokio-postgres"] } +# Database - PostgreSQL (default, feature-gated) +deadpool-postgres = { version = "0.14", optional = true } +tokio-postgres = { version = "0.7", features = ["with-uuid-1", "with-chrono-0_4", "with-serde_json-1"], optional = true } +postgres-types = { version = "0.2", features = ["with-serde_json-1"], optional = true } +refinery = { version = "0.8", features = ["tokio-postgres"], optional = true } + +# Database - libSQL/Turso (optional embedded database) +libsql = { version = "0.6", optional = true, default-features = false, features = ["core", "replication"] } # Error handling thiserror = "2" @@ -48,7 +51,7 @@ dotenvy = "0.15" # Core types uuid = { version = "1", features = ["v4", "serde"] } chrono = { version = "0.4", features = ["serde"] } -rust_decimal = { version = "1", features = ["serde", "serde-with-str", "db-tokio-postgres", "maths"] } +rust_decimal = { version = "1", features = ["serde", "serde-with-str", "maths"] } rust_decimal_macros = "1" # Async traits @@ -89,7 +92,7 @@ open = "5" # Vector embeddings for semantic search # The postgres feature provides ToSql/FromSql for postgres-types (shared by tokio-postgres) -pgvector = { version = "0.4", features = ["postgres"] } +pgvector = { version = "0.4", features = ["postgres"], optional = true } # WASM sandbox for untrusted tool execution wasmtime = { version = "28", features = ["component-model"] } @@ -135,9 +138,22 @@ pretty_assertions = "1" tempfile = "3" [features] -default = [] +default = ["postgres"] +postgres = [ + "dep:deadpool-postgres", + "dep:tokio-postgres", + "dep:postgres-types", + "dep:refinery", + "dep:pgvector", + "rust_decimal/db-tokio-postgres", +] +libsql = ["dep:libsql"] integration = [] +[[example]] +name = "test_heartbeat" +required-features = ["postgres"] + # The profile that 'cargo dist' will build with [profile.dist] inherits = "release" diff --git a/src/agent/agent_loop.rs b/src/agent/agent_loop.rs index 18a7d6fc..fa9cfb9a 100644 --- a/src/agent/agent_loop.rs +++ b/src/agent/agent_loop.rs @@ -19,9 +19,9 @@ use crate::channels::{ChannelManager, IncomingMessage, OutgoingResponse, StatusU use crate::config::{AgentConfig, HeartbeatConfig, RoutineConfig}; use crate::context::ContextManager; use crate::context::JobContext; +use crate::db::Database; use crate::error::Error; use crate::extensions::ExtensionManager; -use crate::history::Store; use crate::llm::{ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult}; use crate::safety::SafetyLayer; use crate::tools::ToolRegistry; @@ -65,7 +65,7 @@ enum AgenticLoopResult { /// /// Bundles the shared components to reduce argument count. pub struct AgentDeps { - pub store: Option>, + pub store: Option>, pub llm: Arc, pub safety: Arc, pub tools: Arc, @@ -130,7 +130,7 @@ impl Agent { } // Convenience accessors - fn store(&self) -> Option<&Arc> { + fn store(&self) -> Option<&Arc> { self.deps.store.as_ref() } diff --git a/src/agent/routine_engine.rs b/src/agent/routine_engine.rs index e88a0a9d..52156ac5 100644 --- a/src/agent/routine_engine.rs +++ b/src/agent/routine_engine.rs @@ -24,14 +24,14 @@ use crate::agent::routine::{ }; use crate::channels::{IncomingMessage, OutgoingResponse}; use crate::config::RoutineConfig; -use crate::history::Store; +use crate::db::Database; use crate::llm::{ChatMessage, CompletionRequest, FinishReason, LlmProvider}; use crate::workspace::Workspace; /// The routine execution engine. pub struct RoutineEngine { config: RoutineConfig, - store: Arc, + store: Arc, llm: Arc, workspace: Arc, /// Sender for notifications (routed to channel manager). @@ -45,7 +45,7 @@ pub struct RoutineEngine { impl RoutineEngine { pub fn new( config: RoutineConfig, - store: Arc, + store: Arc, llm: Arc, workspace: Arc, notify_tx: mpsc::Sender, @@ -293,7 +293,7 @@ impl RoutineEngine { /// Shared context passed to the execution function. struct EngineContext { - store: Arc, + store: Arc, llm: Arc, workspace: Arc, notify_tx: mpsc::Sender, diff --git a/src/agent/scheduler.rs b/src/agent/scheduler.rs index f37bba28..a665c2af 100644 --- a/src/agent/scheduler.rs +++ b/src/agent/scheduler.rs @@ -12,8 +12,8 @@ use crate::agent::task::{Task, TaskContext, TaskOutput}; use crate::agent::worker::{Worker, WorkerDeps}; use crate::config::AgentConfig; use crate::context::{ContextManager, JobContext, JobState}; +use crate::db::Database; use crate::error::{Error, JobError}; -use crate::history::Store; use crate::llm::LlmProvider; use crate::safety::SafetyLayer; use crate::tools::ToolRegistry; @@ -48,7 +48,7 @@ pub struct Scheduler { llm: Arc, safety: Arc, tools: Arc, - store: Option>, + store: Option>, /// Running jobs (main LLM-driven jobs). jobs: Arc>>, /// Running sub-tasks (tool executions, background tasks). @@ -63,7 +63,7 @@ impl Scheduler { llm: Arc, safety: Arc, tools: Arc, - store: Option>, + store: Option>, ) -> Self { Self { config, diff --git a/src/agent/self_repair.rs b/src/agent/self_repair.rs index ace75fe8..ee7b2a4c 100644 --- a/src/agent/self_repair.rs +++ b/src/agent/self_repair.rs @@ -8,8 +8,8 @@ use chrono::{DateTime, Utc}; use uuid::Uuid; use crate::context::{ContextManager, JobState}; +use crate::db::Database; use crate::error::RepairError; -use crate::history::Store; use crate::tools::{BuildRequirement, Language, SoftwareBuilder, SoftwareType, ToolRegistry}; /// A job that has been detected as stuck. @@ -69,7 +69,7 @@ pub struct DefaultSelfRepair { #[allow(dead_code)] // Will be used for time-based stuck detection stuck_threshold: Duration, max_repair_attempts: u32, - store: Option>, + store: Option>, builder: Option>, #[allow(dead_code)] // Will be used for tool hot-reload after repair tools: Option>, @@ -94,7 +94,7 @@ impl DefaultSelfRepair { /// Add a Store for tool failure tracking. #[allow(dead_code)] // Public API for configuring repair with persistence - pub fn with_store(mut self, store: Arc) -> Self { + pub fn with_store(mut self, store: Arc) -> Self { self.store = Some(store); self } diff --git a/src/agent/worker.rs b/src/agent/worker.rs index bf8ec895..39c5cd8d 100644 --- a/src/agent/worker.rs +++ b/src/agent/worker.rs @@ -10,8 +10,8 @@ use uuid::Uuid; use crate::agent::scheduler::WorkerMessage; use crate::agent::task::TaskOutput; use crate::context::{ContextManager, JobState}; +use crate::db::Database; use crate::error::Error; -use crate::history::Store; use crate::llm::{ ActionPlan, ChatMessage, LlmProvider, Reasoning, ReasoningContext, RespondResult, ToolSelection, }; @@ -28,7 +28,7 @@ pub struct WorkerDeps { pub llm: Arc, pub safety: Arc, pub tools: Arc, - pub store: Option>, + pub store: Option>, pub timeout: Duration, pub use_planning: bool, } @@ -67,7 +67,7 @@ impl Worker { &self.deps.tools } - fn store(&self) -> Option<&Arc> { + fn store(&self) -> Option<&Arc> { self.deps.store.as_ref() } @@ -381,7 +381,7 @@ Report when the job is complete or if you encounter issues you cannot resolve."# tools: Arc, context_manager: Arc, safety: Arc, - store: Option>, + store: Option>, job_id: Uuid, tool_name: &str, params: &serde_json::Value, diff --git a/src/bootstrap.rs b/src/bootstrap.rs index 72ff65c0..e24b996e 100644 --- a/src/bootstrap.rs +++ b/src/bootstrap.rs @@ -124,7 +124,7 @@ impl BootstrapConfig { /// If both conditions hold, migrates settings, MCP servers, and session data /// to the database, writes `bootstrap.json`, and renames old files to `.migrated`. pub async fn migrate_disk_to_db( - store: &crate::history::Store, + store: &dyn crate::db::Database, user_id: &str, ) -> Result<(), MigrationError> { let legacy_settings_path = BootstrapConfig::legacy_settings_path(); diff --git a/src/channels/web/mod.rs b/src/channels/web/mod.rs index ac6ef2f6..356eda2c 100644 --- a/src/channels/web/mod.rs +++ b/src/channels/web/mod.rs @@ -32,9 +32,9 @@ use tokio_stream::wrappers::ReceiverStream; use crate::agent::SessionManager; use crate::channels::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate}; use crate::config::GatewayConfig; +use crate::db::Database; use crate::error::ChannelError; use crate::extensions::ExtensionManager; -use crate::history::Store; use crate::orchestrator::job_manager::ContainerJobManager; use crate::tools::ToolRegistry; use crate::workspace::Workspace; @@ -147,7 +147,7 @@ impl GatewayChannel { } /// Inject the database store for sandbox job persistence. - pub fn with_store(mut self, store: Arc) -> Self { + pub fn with_store(mut self, store: Arc) -> Self { self.rebuild_state(|s| s.store = Some(store)); self } diff --git a/src/channels/web/server.rs b/src/channels/web/server.rs index 68bf6edf..744c369e 100644 --- a/src/channels/web/server.rs +++ b/src/channels/web/server.rs @@ -30,8 +30,8 @@ use crate::channels::web::auth::{AuthState, auth_middleware}; use crate::channels::web::log_layer::LogBroadcaster; use crate::channels::web::sse::SseManager; use crate::channels::web::types::*; +use crate::db::Database; use crate::extensions::ExtensionManager; -use crate::history::Store; use crate::orchestrator::job_manager::ContainerJobManager; use crate::tools::ToolRegistry; use crate::workspace::Workspace; @@ -126,7 +126,7 @@ pub struct GatewayState { /// Tool registry for listing registered tools. pub tool_registry: Option>, /// Database store for sandbox job persistence. - pub store: Option>, + pub store: Option>, /// Container job manager for sandbox operations. pub job_manager: Option>, /// Prompt queue for Claude Code follow-up prompts. diff --git a/src/cli/config.rs b/src/cli/config.rs index 07788dca..8835b2c0 100644 --- a/src/cli/config.rs +++ b/src/cli/config.rs @@ -1,7 +1,9 @@ //! Configuration management CLI commands. //! //! Commands for viewing and modifying settings. -//! Settings are stored in PostgreSQL (env > DB > default). +//! Settings are stored in the database (env > DB > default). + +use std::sync::Arc; use clap::Subcommand; @@ -49,8 +51,8 @@ pub async fn run_config_command(cmd: ConfigCommand) -> anyhow::Result<()> { let _ = dotenvy::dotenv(); // Try to connect to the DB for settings access - let store = match connect_store().await { - Ok(s) => Some(s), + let db: Option> = match connect_db().await { + Ok(d) => Some(d), Err(e) => { eprintln!( "Warning: Could not connect to database ({}), using disk fallback", @@ -60,29 +62,30 @@ pub async fn run_config_command(cmd: ConfigCommand) -> anyhow::Result<()> { } }; + let db_ref = db.as_deref(); match cmd { - ConfigCommand::List { filter } => list_settings(store.as_ref(), filter).await, - ConfigCommand::Get { path } => get_setting(store.as_ref(), &path).await, - ConfigCommand::Set { path, value } => set_setting(store.as_ref(), &path, &value).await, - ConfigCommand::Reset { path } => reset_setting(store.as_ref(), &path).await, - ConfigCommand::Path => show_path(store.is_some()), + ConfigCommand::List { filter } => list_settings(db_ref, filter).await, + ConfigCommand::Get { path } => get_setting(db_ref, &path).await, + ConfigCommand::Set { path, value } => set_setting(db_ref, &path, &value).await, + ConfigCommand::Reset { path } => reset_setting(db_ref, &path).await, + ConfigCommand::Path => show_path(db_ref.is_some()), } } -/// Bootstrap a DB connection for config commands. -async fn connect_store() -> anyhow::Result { +/// Bootstrap a DB connection for config commands (backend-agnostic). +async fn connect_db() -> anyhow::Result> { let config = crate::config::Config::from_env() .await .map_err(|e| anyhow::anyhow!("{}", e))?; - let store = crate::history::Store::new(&config.database).await?; - store.run_migrations().await?; - Ok(store) + crate::db::connect_from_config(&config.database) + .await + .map_err(|e| anyhow::anyhow!("{}", e)) } const DEFAULT_USER_ID: &str = "default"; /// Load settings: DB if available, else disk. -async fn load_settings(store: Option<&crate::history::Store>) -> Settings { +async fn load_settings(store: Option<&dyn crate::db::Database>) -> Settings { if let Some(store) = store { match store.get_all_settings(DEFAULT_USER_ID).await { Ok(map) if !map.is_empty() => return Settings::from_db_map(&map), @@ -94,7 +97,7 @@ async fn load_settings(store: Option<&crate::history::Store>) -> Settings { /// List all settings. async fn list_settings( - store: Option<&crate::history::Store>, + store: Option<&dyn crate::db::Database>, filter: Option, ) -> anyhow::Result<()> { let settings = load_settings(store).await; @@ -126,7 +129,7 @@ async fn list_settings( } /// Get a specific setting. -async fn get_setting(store: Option<&crate::history::Store>, path: &str) -> anyhow::Result<()> { +async fn get_setting(store: Option<&dyn crate::db::Database>, path: &str) -> anyhow::Result<()> { let settings = load_settings(store).await; match settings.get(path) { @@ -142,7 +145,7 @@ async fn get_setting(store: Option<&crate::history::Store>, path: &str) -> anyho /// Set a setting value. async fn set_setting( - store: Option<&crate::history::Store>, + store: Option<&dyn crate::db::Database>, path: &str, value: &str, ) -> anyhow::Result<()> { @@ -171,7 +174,7 @@ async fn set_setting( } /// Reset a setting to default. -async fn reset_setting(store: Option<&crate::history::Store>, path: &str) -> anyhow::Result<()> { +async fn reset_setting(store: Option<&dyn crate::db::Database>, path: &str) -> anyhow::Result<()> { let default = Settings::default(); let default_value = default .get(path) @@ -196,7 +199,7 @@ async fn reset_setting(store: Option<&crate::history::Store>, path: &str) -> any /// Show the settings storage info. fn show_path(has_db: bool) -> anyhow::Result<()> { if has_db { - println!("Settings stored in: PostgreSQL (settings table)"); + println!("Settings stored in: database (settings table)"); println!( "Bootstrap config: {}", crate::bootstrap::BootstrapConfig::default_path().display() diff --git a/src/cli/mcp.rs b/src/cli/mcp.rs index 4600e302..e61b65de 100644 --- a/src/cli/mcp.rs +++ b/src/cli/mcp.rs @@ -8,8 +8,10 @@ use std::sync::Arc; use clap::Subcommand; use crate::config::Config; -use crate::history::Store; -use crate::secrets::{PostgresSecretsStore, SecretsCrypto, SecretsStore}; +use crate::db::Database; +#[cfg(feature = "postgres")] +use crate::secrets::PostgresSecretsStore; +use crate::secrets::{SecretsCrypto, SecretsStore}; use crate::tools::mcp::{ McpClient, McpServerConfig, McpSessionManager, OAuthConfig, auth::{authorize_mcp_server, is_authenticated}, @@ -172,10 +174,10 @@ async fn add_server( config.validate()?; // Save (DB if available, else disk) - let store = connect_store().await; - let mut servers = load_servers(store.as_ref()).await?; + let db = connect_db().await; + let mut servers = load_servers(db.as_deref()).await?; servers.upsert(config); - save_servers(store.as_ref(), &servers).await?; + save_servers(db.as_deref(), &servers).await?; println!(); println!(" ✓ Added MCP server '{}'", name); @@ -193,12 +195,12 @@ async fn add_server( /// Remove an MCP server. async fn remove_server(name: String) -> anyhow::Result<()> { - let store = connect_store().await; - let mut servers = load_servers(store.as_ref()).await?; + let db = connect_db().await; + let mut servers = load_servers(db.as_deref()).await?; if !servers.remove(&name) { anyhow::bail!("Server '{}' not found", name); } - save_servers(store.as_ref(), &servers).await?; + save_servers(db.as_deref(), &servers).await?; println!(); println!(" ✓ Removed MCP server '{}'", name); @@ -209,8 +211,8 @@ async fn remove_server(name: String) -> anyhow::Result<()> { /// List configured MCP servers. async fn list_servers(verbose: bool) -> anyhow::Result<()> { - let store = connect_store().await; - let servers = load_servers(store.as_ref()).await?; + let db = connect_db().await; + let servers = load_servers(db.as_deref()).await?; if servers.servers.is_empty() { println!(); @@ -268,8 +270,8 @@ async fn list_servers(verbose: bool) -> anyhow::Result<()> { /// Authenticate with an MCP server. async fn auth_server(name: String, user_id: String) -> anyhow::Result<()> { // Get server config - let store = connect_store().await; - let servers = load_servers(store.as_ref()).await?; + let db = connect_db().await; + let servers = load_servers(db.as_deref()).await?; let server = servers .get(&name) .cloned() @@ -341,8 +343,8 @@ async fn auth_server(name: String, user_id: String) -> anyhow::Result<()> { /// Test connection to an MCP server. async fn test_server(name: String, user_id: String) -> anyhow::Result<()> { // Get server config - let store = connect_store().await; - let servers = load_servers(store.as_ref()).await?; + let db = connect_db().await; + let servers = load_servers(db.as_deref()).await?; let server = servers .get(&name) .cloned() @@ -437,8 +439,8 @@ async fn test_server(name: String, user_id: String) -> anyhow::Result<()> { /// Toggle server enabled/disabled state. async fn toggle_server(name: String, enable: bool, disable: bool) -> anyhow::Result<()> { - let store = connect_store().await; - let mut servers = load_servers(store.as_ref()).await?; + let db = connect_db().await; + let mut servers = load_servers(db.as_deref()).await?; let server = servers .get_mut(&name) @@ -453,7 +455,7 @@ async fn toggle_server(name: String, enable: bool, disable: bool) -> anyhow::Res }; server.enabled = new_state; - save_servers(store.as_ref(), &servers).await?; + save_servers(db.as_deref(), &servers).await?; let status = if new_state { "enabled" } else { "disabled" }; println!(); @@ -465,18 +467,16 @@ async fn toggle_server(name: String, enable: bool, disable: bool) -> anyhow::Res const DEFAULT_USER_ID: &str = "default"; -/// Try to connect to the database store for DB-backed config. -async fn connect_store() -> Option { +/// Try to connect to the database (backend-agnostic). +async fn connect_db() -> Option> { let config = Config::from_env().await.ok()?; - let store = Store::new(&config.database).await.ok()?; - store.run_migrations().await.ok()?; - Some(store) + crate::db::connect_from_config(&config.database).await.ok() } /// Load MCP servers (DB if available, else disk). -async fn load_servers(store: Option<&Store>) -> Result { - if let Some(store) = store { - config::load_mcp_servers_from_db(store, DEFAULT_USER_ID).await +async fn load_servers(db: Option<&dyn Database>) -> Result { + if let Some(db) = db { + config::load_mcp_servers_from_db(db, DEFAULT_USER_ID).await } else { config::load_mcp_servers().await } @@ -484,11 +484,11 @@ async fn load_servers(store: Option<&Store>) -> Result, + db: Option<&dyn Database>, servers: &McpServersFile, ) -> Result<(), config::ConfigError> { - if let Some(store) = store { - config::save_mcp_servers_to_db(store, DEFAULT_USER_ID, servers).await + if let Some(db) = db { + config::save_mcp_servers_to_db(db, DEFAULT_USER_ID, servers).await } else { config::save_mcp_servers(servers).await } @@ -504,14 +504,61 @@ async fn get_secrets_store() -> anyhow::Result, + embeddings: Option>, +) -> anyhow::Result<()> { + let mut workspace = Workspace::new_with_db("default", db); + if let Some(emb) = embeddings { + workspace = workspace.with_embeddings(emb); + } + + match cmd { + MemoryCommand::Search { query, limit } => search(&workspace, &query, limit).await, + MemoryCommand::Read { path } => read(&workspace, &path).await, + MemoryCommand::Write { + path, + content, + append, + } => write(&workspace, &path, content, append).await, + MemoryCommand::Tree { path, depth } => tree(&workspace, &path, depth).await, + MemoryCommand::Status => status(&workspace).await, + } +} + #[derive(Subcommand, Debug, Clone)] pub enum MemoryCommand { /// Search workspace memory (hybrid full-text + semantic) @@ -55,7 +79,8 @@ pub enum MemoryCommand { Status, } -/// Run a memory command. +/// Run a memory command (PostgreSQL backend). +#[cfg(feature = "postgres")] pub async fn run_memory_command( cmd: MemoryCommand, pool: deadpool_postgres::Pool, diff --git a/src/cli/mod.rs b/src/cli/mod.rs index 5823ab68..77ed1f3d 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -18,7 +18,10 @@ mod tool; pub use config::{ConfigCommand, run_config_command}; pub use mcp::{McpCommand, run_mcp_command}; -pub use memory::{MemoryCommand, run_memory_command}; +pub use memory::MemoryCommand; +#[cfg(feature = "postgres")] +pub use memory::run_memory_command; +pub use memory::run_memory_command_with_db; pub use pairing::{PairingCommand, run_pairing_command, run_pairing_command_with_store}; pub use status::run_status_command; pub use tool::{ToolCommand, run_tool_command}; diff --git a/src/cli/status.rs b/src/cli/status.rs index 0db48c90..af4585d8 100644 --- a/src/cli/status.rs +++ b/src/cli/status.rs @@ -135,6 +135,7 @@ pub async fn run_status_command() -> anyhow::Result<()> { Ok(()) } +#[cfg(feature = "postgres")] async fn check_database() -> anyhow::Result<()> { let _ = dotenvy::dotenv(); let settings = Settings::load(); @@ -167,6 +168,12 @@ async fn check_database() -> anyhow::Result<()> { Ok(()) } +#[cfg(not(feature = "postgres"))] +async fn check_database() -> anyhow::Result<()> { + // For non-postgres backends, just report configured + Ok(()) +} + fn count_wasm_files(dir: &std::path::Path) -> usize { std::fs::read_dir(dir) .map(|entries| { diff --git a/src/cli/tool.rs b/src/cli/tool.rs index f49c3a8d..299225b8 100644 --- a/src/cli/tool.rs +++ b/src/cli/tool.rs @@ -11,8 +11,11 @@ use clap::Subcommand; use tokio::fs; use crate::config::Config; -use crate::history::Store; -use crate::secrets::{CreateSecretParams, PostgresSecretsStore, SecretsCrypto, SecretsStore}; +#[allow(unused_imports)] +use crate::db::Database; +#[cfg(feature = "postgres")] +use crate::secrets::PostgresSecretsStore; +use crate::secrets::{CreateSecretParams, SecretsCrypto, SecretsStore}; use crate::tools::wasm::{CapabilitiesFile, compute_binary_hash}; /// Default tools directory. @@ -722,11 +725,58 @@ async fn auth_tool(name: String, dir: Option, user_id: String) -> anyho ) })?; - let store = Store::new(&config.database).await?; - store.run_migrations().await?; - let crypto = SecretsCrypto::new(master_key.clone())?; - let secrets_store = Arc::new(PostgresSecretsStore::new(store.pool(), Arc::new(crypto))); + + let secrets_store: Arc = { + #[cfg(feature = "postgres")] + { + let store = crate::history::Store::new(&config.database).await?; + store.run_migrations().await?; + Arc::new(PostgresSecretsStore::new(store.pool(), Arc::new(crypto))) + } + #[cfg(all(feature = "libsql", not(feature = "postgres")))] + { + use crate::db::Database as _; + use crate::db::libsql_backend::LibSqlBackend; + use secrecy::ExposeSecret as _; + + let default_path = crate::config::default_libsql_path(); + let db_path = config + .database + .libsql_path + .as_deref() + .unwrap_or(&default_path); + + let backend = if let Some(ref url) = config.database.libsql_url { + let token = config.database.libsql_auth_token.as_ref().ok_or_else(|| { + anyhow::anyhow!("LIBSQL_AUTH_TOKEN is required when LIBSQL_URL is set") + })?; + LibSqlBackend::new_remote_replica(db_path, url, token.expose_secret()) + .await + .map_err(|e| anyhow::anyhow!("{}", e))? + } else { + LibSqlBackend::new_local(db_path) + .await + .map_err(|e| anyhow::anyhow!("{}", e))? + }; + backend + .run_migrations() + .await + .map_err(|e| anyhow::anyhow!("{}", e))?; + + Arc::new(crate::secrets::LibSqlSecretsStore::new( + backend.shared_db(), + Arc::new(crypto), + )) + } + #[cfg(not(any(feature = "postgres", feature = "libsql")))] + { + let _ = crypto; + anyhow::bail!( + "No database backend available for secrets. Enable 'postgres' or 'libsql' feature." + ); + } + }; // Check if already configured let already_configured = secrets_store @@ -773,29 +823,29 @@ async fn auth_tool(name: String, dir: Option, user_id: String) -> anyho println!(" Validation failed: {}", e); println!(); println!(" Falling back to manual entry..."); - return auth_tool_manual(&secrets_store, &user_id, &auth).await; + return auth_tool_manual(secrets_store.as_ref(), &user_id, &auth).await; } } } // Save the token - save_token(&secrets_store, &user_id, &auth, &token).await?; + save_token(secrets_store.as_ref(), &user_id, &auth, &token).await?; print_success(display_name); return Ok(()); } // Check for OAuth configuration if let Some(ref oauth) = auth.oauth { - return auth_tool_oauth(&secrets_store, &user_id, &auth, oauth).await; + return auth_tool_oauth(secrets_store.as_ref(), &user_id, &auth, oauth).await; } // Fall back to manual entry - auth_tool_manual(&secrets_store, &user_id, &auth).await + auth_tool_manual(secrets_store.as_ref(), &user_id, &auth).await } /// OAuth browser-based login flow. async fn auth_tool_oauth( - store: &PostgresSecretsStore, + store: &(dyn SecretsStore + Send + Sync), user_id: &str, auth: &crate::tools::wasm::AuthCapabilitySchema, oauth: &crate::tools::wasm::OAuthConfigSchema, @@ -1041,7 +1091,7 @@ async fn auth_tool_oauth( /// Manual token entry flow. async fn auth_tool_manual( - store: &PostgresSecretsStore, + store: &(dyn SecretsStore + Send + Sync), user_id: &str, auth: &crate::tools::wasm::AuthCapabilitySchema, ) -> anyhow::Result<()> { @@ -1214,7 +1264,7 @@ async fn validate_token( /// Save token to secrets store. async fn save_token( - store: &PostgresSecretsStore, + store: &(dyn SecretsStore + Send + Sync), user_id: &str, auth: &crate::tools::wasm::AuthCapabilitySchema, token: &str, diff --git a/src/config.rs b/src/config.rs index 285a2914..a6ddb158 100644 --- a/src/config.rs +++ b/src/config.rs @@ -38,7 +38,7 @@ impl Config { /// Priority: env var > DB settings > default. /// This is the primary way to load config after DB is connected. pub async fn from_db( - store: &crate::history::Store, + store: &dyn crate::db::Database, user_id: &str, bootstrap: &crate::bootstrap::BootstrapConfig, ) -> Result { @@ -134,17 +134,72 @@ impl TunnelConfig { } } +/// Which database backend to use. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum DatabaseBackend { + /// PostgreSQL via deadpool-postgres (default). + #[default] + Postgres, + /// libSQL/Turso embedded database. + LibSql, +} + +impl std::str::FromStr for DatabaseBackend { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "postgres" | "postgresql" | "pg" => Ok(Self::Postgres), + "libsql" | "turso" | "sqlite" => Ok(Self::LibSql), + _ => Err(format!( + "invalid database backend '{}', expected 'postgres' or 'libsql'", + s + )), + } + } +} + /// Database configuration. #[derive(Debug, Clone)] pub struct DatabaseConfig { + /// Which backend to use (default: Postgres). + pub backend: DatabaseBackend, + + // -- PostgreSQL fields -- pub url: SecretString, pub pool_size: usize, + + // -- libSQL fields -- + /// Path to local libSQL database file (default: ~/.ironclaw/ironclaw.db). + pub libsql_path: Option, + /// Turso cloud URL for remote sync (optional). + pub libsql_url: Option, + /// Turso auth token (required when libsql_url is set). + pub libsql_auth_token: Option, } impl DatabaseConfig { fn resolve(bootstrap: &crate::bootstrap::BootstrapConfig) -> Result { + let backend: DatabaseBackend = if let Some(b) = optional_env("DATABASE_BACKEND")? { + b.parse().map_err(|e| ConfigError::InvalidValue { + key: "DATABASE_BACKEND".to_string(), + message: e, + })? + } else { + DatabaseBackend::default() + }; + + // PostgreSQL URL is required only when using the postgres backend. + // For libsql backend, default to an empty placeholder. let url = optional_env("DATABASE_URL")? .or_else(|| bootstrap.database_url.clone()) + .or_else(|| { + if backend == DatabaseBackend::LibSql { + Some("unused://libsql".to_string()) + } else { + None + } + }) .ok_or_else(|| ConfigError::MissingRequired { key: "database_url".to_string(), hint: "Run 'ironclaw onboard' or set DATABASE_URL environment variable".to_string(), @@ -160,9 +215,31 @@ impl DatabaseConfig { .or(bootstrap.database_pool_size) .unwrap_or(10); + let libsql_path = optional_env("LIBSQL_PATH")?.map(PathBuf::from).or_else(|| { + if backend == DatabaseBackend::LibSql { + Some(default_libsql_path()) + } else { + None + } + }); + + let libsql_url = optional_env("LIBSQL_URL")?; + let libsql_auth_token = optional_env("LIBSQL_AUTH_TOKEN")?.map(SecretString::from); + + if libsql_url.is_some() && libsql_auth_token.is_none() { + return Err(ConfigError::MissingRequired { + key: "LIBSQL_AUTH_TOKEN".to_string(), + hint: "LIBSQL_AUTH_TOKEN is required when LIBSQL_URL is set".to_string(), + }); + } + Ok(Self { + backend, url: SecretString::from(url), pool_size, + libsql_path, + libsql_url, + libsql_auth_token, }) } @@ -172,6 +249,14 @@ impl DatabaseConfig { } } +/// Default libSQL database path (~/.ironclaw/ironclaw.db). +pub fn default_libsql_path() -> PathBuf { + dirs::home_dir() + .unwrap_or_else(|| PathBuf::from(".")) + .join(".ironclaw") + .join("ironclaw.db") +} + /// Which LLM backend to use. /// /// Defaults to `NearAi` to keep IronClaw close to the NEAR ecosystem. diff --git a/src/db/libsql_backend.rs b/src/db/libsql_backend.rs new file mode 100644 index 00000000..8dde3ad3 --- /dev/null +++ b/src/db/libsql_backend.rs @@ -0,0 +1,2609 @@ +//! libSQL/Turso backend for the Database trait. +//! +//! Provides an embedded SQLite-compatible database using Turso's libSQL fork. +//! Supports three modes: +//! - Local embedded (file-based, no server needed) +//! - Turso cloud with embedded replica (sync to cloud) +//! - In-memory (for testing) + +use std::collections::HashMap; +use std::path::Path; +use std::sync::Arc; + +use async_trait::async_trait; +use chrono::{DateTime, NaiveDateTime, Utc}; +use libsql::{Connection, Database as LibSqlDatabase, params}; +use rust_decimal::Decimal; +use uuid::Uuid; + +use crate::agent::BrokenTool; +use crate::agent::routine::{ + NotifyConfig, Routine, RoutineAction, RoutineGuardrails, RoutineRun, RunStatus, Trigger, +}; +use crate::context::{ActionRecord, JobContext, JobState}; +use crate::db::Database; +use crate::error::{DatabaseError, WorkspaceError}; +use crate::history::{ + ConversationMessage, ConversationSummary, JobEventRecord, LlmCallRecord, SandboxJobRecord, + SandboxJobSummary, SettingRow, +}; +use crate::workspace::{ + MemoryChunk, MemoryDocument, RankedResult, SearchConfig, SearchResult, WorkspaceEntry, + reciprocal_rank_fusion, +}; + +use crate::db::libsql_migrations; + +/// Explicit column list for routines table (matches positional access in `row_to_routine_libsql`). +const ROUTINE_COLUMNS: &str = "\ + id, name, description, user_id, enabled, \ + trigger_type, trigger_config, action_type, action_config, \ + cooldown_secs, max_concurrent, dedup_window_secs, \ + notify_channel, notify_user, notify_on_success, notify_on_failure, notify_on_attention, \ + state, last_run_at, next_fire_at, run_count, consecutive_failures, \ + created_at, updated_at"; + +/// Explicit column list for routine_runs table (matches positional access in `row_to_routine_run_libsql`). +const ROUTINE_RUN_COLUMNS: &str = "\ + id, routine_id, trigger_type, trigger_detail, started_at, \ + status, completed_at, result_summary, tokens_used, job_id, created_at"; + +/// libSQL/Turso database backend. +/// +/// Stores the `Database` handle in an `Arc` so that the same underlying +/// database can be shared with stores (SecretsStore, WasmToolStore) that +/// create their own connections per-operation. +pub struct LibSqlBackend { + db: Arc, +} + +impl LibSqlBackend { + /// Create a new local embedded database. + pub async fn new_local(path: &Path) -> Result { + // Ensure parent directory exists + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| { + DatabaseError::Pool(format!("Failed to create database directory: {}", e)) + })?; + } + + let db = libsql::Builder::new_local(path) + .build() + .await + .map_err(|e| DatabaseError::Pool(format!("Failed to open libSQL database: {}", e)))?; + + Ok(Self { db: Arc::new(db) }) + } + + /// Create a new in-memory database (for testing). + pub async fn new_memory() -> Result { + let db = libsql::Builder::new_local(":memory:") + .build() + .await + .map_err(|e| { + DatabaseError::Pool(format!("Failed to create in-memory database: {}", e)) + })?; + + Ok(Self { db: Arc::new(db) }) + } + + /// Create with Turso cloud sync (embedded replica). + pub async fn new_remote_replica( + path: &Path, + url: &str, + auth_token: &str, + ) -> Result { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| { + DatabaseError::Pool(format!("Failed to create database directory: {}", e)) + })?; + } + + let db = libsql::Builder::new_remote_replica(path, url.to_string(), auth_token.to_string()) + .build() + .await + .map_err(|e| DatabaseError::Pool(format!("Failed to open remote replica: {}", e)))?; + + Ok(Self { db: Arc::new(db) }) + } + + /// Get a shared reference to the underlying database handle. + /// + /// Use this to pass the database to stores (SecretsStore, WasmToolStore) + /// that need to create their own connections per-operation. + pub fn shared_db(&self) -> Arc { + Arc::clone(&self.db) + } + + /// Create a new connection to the database. + pub fn connect(&self) -> Result { + self.db + .connect() + .map_err(|e| DatabaseError::Pool(format!("Failed to create connection: {}", e))) + } +} + +// ==================== Helper functions ==================== + +/// Parse an ISO-8601 timestamp string from SQLite into DateTime. +/// +/// Tries multiple formats in order: +/// 1. RFC 3339 with timezone (e.g. `2024-01-15T10:30:00.123Z`) +/// 2. Naive datetime with fractional seconds (e.g. `2024-01-15 10:30:00.123`) +/// 3. Naive datetime without fractional seconds (e.g. `2024-01-15 10:30:00`) +/// +/// Returns an error if none of the formats match. +fn parse_timestamp(s: &str) -> Result, String> { + // RFC 3339 (our canonical write format) + if let Ok(dt) = DateTime::parse_from_rfc3339(s) { + return Ok(dt.with_timezone(&Utc)); + } + // Naive with fractional seconds (legacy or SQLite datetime() output) + if let Ok(ndt) = NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f") { + return Ok(ndt.and_utc()); + } + // Naive without fractional seconds (legacy format) + if let Ok(ndt) = NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S") { + return Ok(ndt.and_utc()); + } + Err(format!("unparseable timestamp: {:?}", s)) +} + +/// Format a DateTime for SQLite storage (RFC 3339 with millisecond precision). +fn fmt_ts(dt: &DateTime) -> String { + dt.to_rfc3339_opts(chrono::SecondsFormat::Millis, true) +} + +/// Format an optional DateTime. +fn fmt_opt_ts(dt: &Option>) -> libsql::Value { + match dt { + Some(dt) => libsql::Value::Text(fmt_ts(dt)), + None => libsql::Value::Null, + } +} + +fn parse_job_state(s: &str) -> JobState { + match s { + "pending" => JobState::Pending, + "in_progress" => JobState::InProgress, + "completed" => JobState::Completed, + "submitted" => JobState::Submitted, + "accepted" => JobState::Accepted, + "failed" => JobState::Failed, + "stuck" => JobState::Stuck, + "cancelled" => JobState::Cancelled, + _ => JobState::Pending, + } +} + +/// Extract a text column from a libsql Row, returning empty string for NULL. +fn get_text(row: &libsql::Row, idx: i32) -> String { + row.get::(idx).unwrap_or_default() +} + +/// Extract an optional text column. +/// Returns None for SQL NULL, preserves empty strings as Some(""). +fn get_opt_text(row: &libsql::Row, idx: i32) -> Option { + row.get::(idx).ok() +} + +/// Convert an `Option<&str>` to a `libsql::Value` (Text or Null). +/// Use this instead of `.unwrap_or("")` to preserve NULL semantics. +fn opt_text(s: Option<&str>) -> libsql::Value { + match s { + Some(s) => libsql::Value::Text(s.to_string()), + None => libsql::Value::Null, + } +} + +/// Convert an `Option` to a `libsql::Value` (Text or Null). +fn opt_text_owned(s: Option) -> libsql::Value { + match s { + Some(s) => libsql::Value::Text(s), + None => libsql::Value::Null, + } +} + +/// Extract an i64 column, defaulting to 0. +fn get_i64(row: &libsql::Row, idx: i32) -> i64 { + row.get::(idx).unwrap_or(0) +} + +/// Extract an optional bool from an integer column. +fn get_opt_bool(row: &libsql::Row, idx: i32) -> Option { + row.get::(idx).ok().map(|v| v != 0) +} + +/// Parse a Decimal from a text column. +fn get_decimal(row: &libsql::Row, idx: i32) -> Decimal { + row.get::(idx) + .ok() + .and_then(|s| s.parse::().ok()) + .unwrap_or_default() +} + +/// Parse an optional Decimal from a text column. +fn get_opt_decimal(row: &libsql::Row, idx: i32) -> Option { + row.get::(idx) + .ok() + .and_then(|s| s.parse::().ok()) +} + +/// Parse a JSON value from a text column. +fn get_json(row: &libsql::Row, idx: i32) -> serde_json::Value { + row.get::(idx) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()) + .unwrap_or(serde_json::Value::Null) +} + +/// Parse a timestamp from a text column. +/// +/// If the column is NULL or the value cannot be parsed, logs a warning and +/// returns the Unix epoch (1970-01-01T00:00:00Z) so the error is detectable +/// rather than silently replaced by the current time. +fn get_ts(row: &libsql::Row, idx: i32) -> DateTime { + match row.get::(idx) { + Ok(s) => match parse_timestamp(&s) { + Ok(dt) => dt, + Err(e) => { + tracing::warn!("Timestamp parse failure at column {}: {}", idx, e); + DateTime::UNIX_EPOCH + } + }, + Err(_) => DateTime::UNIX_EPOCH, + } +} + +/// Parse an optional timestamp from a text column. +/// +/// Returns None if the column is NULL. Logs a warning and returns None if the +/// value is present but cannot be parsed. +fn get_opt_ts(row: &libsql::Row, idx: i32) -> Option> { + match row.get::(idx) { + Ok(s) if s.is_empty() => None, + Ok(s) => match parse_timestamp(&s) { + Ok(dt) => Some(dt), + Err(e) => { + tracing::warn!("Timestamp parse failure at column {}: {}", idx, e); + None + } + }, + Err(_) => None, + } +} + +#[async_trait] +impl Database for LibSqlBackend { + async fn run_migrations(&self) -> Result<(), DatabaseError> { + let conn = self.connect()?; + conn.execute_batch(libsql_migrations::SCHEMA) + .await + .map_err(|e| DatabaseError::Migration(format!("libSQL migration failed: {}", e)))?; + Ok(()) + } + + // ==================== Conversations ==================== + + async fn create_conversation( + &self, + channel: &str, + user_id: &str, + thread_id: Option<&str>, + ) -> Result { + let conn = self.connect()?; + let id = Uuid::new_v4(); + conn.execute( + "INSERT INTO conversations (id, channel, user_id, thread_id) VALUES (?1, ?2, ?3, ?4)", + params![id.to_string(), channel, user_id, opt_text(thread_id)], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(id) + } + + async fn touch_conversation(&self, id: Uuid) -> Result<(), DatabaseError> { + let conn = self.connect()?; + let now = fmt_ts(&Utc::now()); + conn.execute( + "UPDATE conversations SET last_activity = ?2 WHERE id = ?1", + params![id.to_string(), now], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn add_conversation_message( + &self, + conversation_id: Uuid, + role: &str, + content: &str, + ) -> Result { + let conn = self.connect()?; + let id = Uuid::new_v4(); + conn.execute( + "INSERT INTO conversation_messages (id, conversation_id, role, content) VALUES (?1, ?2, ?3, ?4)", + params![id.to_string(), conversation_id.to_string(), role, content], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + self.touch_conversation(conversation_id).await?; + Ok(id) + } + + async fn ensure_conversation( + &self, + id: Uuid, + channel: &str, + user_id: &str, + thread_id: Option<&str>, + ) -> Result<(), DatabaseError> { + let conn = self.connect()?; + let now = fmt_ts(&Utc::now()); + conn.execute( + r#" + INSERT INTO conversations (id, channel, user_id, thread_id) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT (id) DO UPDATE SET last_activity = ?5 + "#, + params![id.to_string(), channel, user_id, opt_text(thread_id), now], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn list_conversations_with_preview( + &self, + user_id: &str, + channel: &str, + limit: i64, + ) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + r#" + SELECT + c.id, + c.started_at, + c.last_activity, + c.metadata, + (SELECT COUNT(*) FROM conversation_messages m WHERE m.conversation_id = c.id) AS message_count, + (SELECT substr(m2.content, 1, 100) + FROM conversation_messages m2 + WHERE m2.conversation_id = c.id AND m2.role = 'user' + ORDER BY m2.created_at ASC + LIMIT 1 + ) AS title + FROM conversations c + WHERE c.user_id = ?1 AND c.channel = ?2 + ORDER BY c.last_activity DESC + LIMIT ?3 + "#, + params![user_id, channel, limit], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut results = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + let metadata = get_json(&row, 3); + let thread_type = metadata + .get("thread_type") + .and_then(|v| v.as_str()) + .map(String::from); + results.push(ConversationSummary { + id: row + .get::(0) + .unwrap_or_default() + .parse() + .unwrap_or_default(), + started_at: get_ts(&row, 1), + last_activity: get_ts(&row, 2), + message_count: get_i64(&row, 4), + title: get_opt_text(&row, 5), + thread_type, + }); + } + Ok(results) + } + + async fn get_or_create_assistant_conversation( + &self, + user_id: &str, + channel: &str, + ) -> Result { + let conn = self.connect()?; + // Try to find existing + let mut rows = conn + .query( + r#" + SELECT id FROM conversations + WHERE user_id = ?1 AND channel = ?2 + AND json_extract(metadata, '$.thread_type') = 'assistant' + LIMIT 1 + "#, + params![user_id, channel], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + if let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + let id_str: String = row.get(0).unwrap_or_default(); + return id_str + .parse() + .map_err(|_| DatabaseError::Serialization("Invalid UUID".to_string())); + } + + // Create new + let id = Uuid::new_v4(); + let metadata = serde_json::json!({"thread_type": "assistant", "title": "Assistant"}); + conn.execute( + "INSERT INTO conversations (id, channel, user_id, metadata) VALUES (?1, ?2, ?3, ?4)", + params![id.to_string(), channel, user_id, metadata.to_string()], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(id) + } + + async fn create_conversation_with_metadata( + &self, + channel: &str, + user_id: &str, + metadata: &serde_json::Value, + ) -> Result { + let conn = self.connect()?; + let id = Uuid::new_v4(); + conn.execute( + "INSERT INTO conversations (id, channel, user_id, metadata) VALUES (?1, ?2, ?3, ?4)", + params![id.to_string(), channel, user_id, metadata.to_string()], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(id) + } + + async fn list_conversation_messages_paginated( + &self, + conversation_id: Uuid, + before: Option>, + limit: i64, + ) -> Result<(Vec, bool), DatabaseError> { + let conn = self.connect()?; + let fetch_limit = limit + 1; + let cid = conversation_id.to_string(); + + let mut rows = if let Some(before_ts) = before { + conn.query( + r#" + SELECT id, role, content, created_at + FROM conversation_messages + WHERE conversation_id = ?1 AND created_at < ?2 + ORDER BY created_at DESC + LIMIT ?3 + "#, + params![cid, fmt_ts(&before_ts), fetch_limit], + ) + .await + } else { + conn.query( + r#" + SELECT id, role, content, created_at + FROM conversation_messages + WHERE conversation_id = ?1 + ORDER BY created_at DESC + LIMIT ?2 + "#, + params![cid, fetch_limit], + ) + .await + } + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut all = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + all.push(ConversationMessage { + id: get_text(&row, 0).parse().unwrap_or_default(), + role: get_text(&row, 1), + content: get_text(&row, 2), + created_at: get_ts(&row, 3), + }); + } + + let has_more = all.len() as i64 > limit; + all.truncate(limit as usize); + all.reverse(); // oldest first + Ok((all, has_more)) + } + + async fn update_conversation_metadata_field( + &self, + id: Uuid, + key: &str, + value: &serde_json::Value, + ) -> Result<(), DatabaseError> { + let conn = self.connect()?; + // SQLite: use json_patch to merge the key + let patch = serde_json::json!({ key: value }); + conn.execute( + "UPDATE conversations SET metadata = json_patch(metadata, ?2) WHERE id = ?1", + params![id.to_string(), patch.to_string()], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn get_conversation_metadata( + &self, + id: Uuid, + ) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + "SELECT metadata FROM conversations WHERE id = ?1", + params![id.to_string()], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + Some(row) => Ok(Some(get_json(&row, 0))), + None => Ok(None), + } + } + + async fn list_conversation_messages( + &self, + conversation_id: Uuid, + ) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + r#" + SELECT id, role, content, created_at + FROM conversation_messages + WHERE conversation_id = ?1 + ORDER BY created_at ASC + "#, + params![conversation_id.to_string()], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut messages = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + messages.push(ConversationMessage { + id: get_text(&row, 0).parse().unwrap_or_default(), + role: get_text(&row, 1), + content: get_text(&row, 2), + created_at: get_ts(&row, 3), + }); + } + Ok(messages) + } + + async fn conversation_belongs_to_user( + &self, + conversation_id: Uuid, + user_id: &str, + ) -> Result { + let conn = self.connect()?; + let mut rows = conn + .query( + "SELECT 1 FROM conversations WHERE id = ?1 AND user_id = ?2", + libsql::params![conversation_id.to_string(), user_id], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + let found = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(found.is_some()) + } + + // ==================== Jobs ==================== + + async fn save_job(&self, ctx: &JobContext) -> Result<(), DatabaseError> { + let conn = self.connect()?; + let status = ctx.state.to_string(); + let estimated_time_secs = ctx.estimated_duration.map(|d| d.as_secs() as i64); + + conn + .execute( + r#" + INSERT INTO agent_jobs ( + id, conversation_id, title, description, category, status, source, + budget_amount, budget_token, bid_amount, estimated_cost, estimated_time_secs, + actual_cost, repair_attempts, created_at, started_at, completed_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17) + ON CONFLICT (id) DO UPDATE SET + title = excluded.title, + description = excluded.description, + category = excluded.category, + status = excluded.status, + estimated_cost = excluded.estimated_cost, + estimated_time_secs = excluded.estimated_time_secs, + actual_cost = excluded.actual_cost, + repair_attempts = excluded.repair_attempts, + started_at = excluded.started_at, + completed_at = excluded.completed_at + "#, + params![ + ctx.job_id.to_string(), + opt_text_owned(ctx.conversation_id.map(|id| id.to_string())), + ctx.title.as_str(), + ctx.description.as_str(), + opt_text(ctx.category.as_deref()), + status, + "direct", + opt_text_owned(ctx.budget.map(|d| d.to_string())), + opt_text(ctx.budget_token.as_deref()), + opt_text_owned(ctx.bid_amount.map(|d| d.to_string())), + opt_text_owned(ctx.estimated_cost.map(|d| d.to_string())), + estimated_time_secs, + ctx.actual_cost.to_string(), + ctx.repair_attempts as i64, + fmt_ts(&ctx.created_at), + fmt_opt_ts(&ctx.started_at), + fmt_opt_ts(&ctx.completed_at), + ], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn get_job(&self, id: Uuid) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + r#" + SELECT id, conversation_id, title, description, category, status, user_id, + budget_amount, budget_token, bid_amount, estimated_cost, estimated_time_secs, + actual_cost, repair_attempts, created_at, started_at, completed_at + FROM agent_jobs WHERE id = ?1 + "#, + params![id.to_string()], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + Some(row) => { + let status_str = get_text(&row, 5); + let state = parse_job_state(&status_str); + let estimated_time_secs: Option = row.get::(11).ok(); + + Ok(Some(JobContext { + job_id: get_text(&row, 0).parse().unwrap_or_default(), + state, + user_id: get_text(&row, 6), + conversation_id: get_opt_text(&row, 1).and_then(|s| s.parse().ok()), + title: get_text(&row, 2), + description: get_text(&row, 3), + category: get_opt_text(&row, 4), + budget: get_opt_decimal(&row, 7), + budget_token: get_opt_text(&row, 8), + bid_amount: get_opt_decimal(&row, 9), + estimated_cost: get_opt_decimal(&row, 10), + estimated_duration: estimated_time_secs + .map(|s| std::time::Duration::from_secs(s as u64)), + actual_cost: get_decimal(&row, 12), + total_tokens_used: 0, + max_tokens: 0, + repair_attempts: get_i64(&row, 13) as u32, + created_at: get_ts(&row, 14), + started_at: get_opt_ts(&row, 15), + completed_at: get_opt_ts(&row, 16), + transitions: Vec::new(), + metadata: serde_json::Value::Null, + })) + } + None => Ok(None), + } + } + + async fn update_job_status( + &self, + id: Uuid, + status: JobState, + failure_reason: Option<&str>, + ) -> Result<(), DatabaseError> { + let conn = self.connect()?; + conn.execute( + "UPDATE agent_jobs SET status = ?2, failure_reason = ?3 WHERE id = ?1", + params![id.to_string(), status.to_string(), opt_text(failure_reason)], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn mark_job_stuck(&self, id: Uuid) -> Result<(), DatabaseError> { + let conn = self.connect()?; + let now = fmt_ts(&Utc::now()); + conn.execute( + "UPDATE agent_jobs SET status = 'stuck', stuck_since = ?2 WHERE id = ?1", + params![id.to_string(), now], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn get_stuck_jobs(&self) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query("SELECT id FROM agent_jobs WHERE status = 'stuck'", ()) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut ids = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + if let Ok(id_str) = row.get::(0) + && let Ok(id) = id_str.parse() + { + ids.push(id); + } + } + Ok(ids) + } + + // ==================== Actions ==================== + + async fn save_action(&self, job_id: Uuid, action: &ActionRecord) -> Result<(), DatabaseError> { + let conn = self.connect()?; + let duration_ms = action.duration.as_millis() as i64; + let warnings_json = serde_json::to_string(&action.sanitization_warnings) + .map_err(|e| DatabaseError::Serialization(e.to_string()))?; + + conn.execute( + r#" + INSERT INTO job_actions ( + id, job_id, sequence_num, tool_name, input, output_raw, output_sanitized, + sanitization_warnings, cost, duration_ms, success, error_message, created_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13) + "#, + params![ + action.id.to_string(), + job_id.to_string(), + action.sequence as i64, + action.tool_name.as_str(), + action.input.to_string(), + opt_text(action.output_raw.as_deref()), + opt_text_owned(action.output_sanitized.as_ref().map(|v| v.to_string())), + warnings_json, + opt_text_owned(action.cost.map(|d| d.to_string())), + duration_ms, + action.success as i64, + opt_text(action.error.as_deref()), + fmt_ts(&action.executed_at), + ], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn get_job_actions(&self, job_id: Uuid) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + r#" + SELECT id, sequence_num, tool_name, input, output_raw, output_sanitized, + sanitization_warnings, cost, duration_ms, success, error_message, created_at + FROM job_actions WHERE job_id = ?1 ORDER BY sequence_num + "#, + params![job_id.to_string()], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut actions = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + let warnings: Vec = + serde_json::from_str(&get_text(&row, 6)).unwrap_or_default(); + actions.push(ActionRecord { + id: get_text(&row, 0).parse().unwrap_or_default(), + sequence: get_i64(&row, 1) as u32, + tool_name: get_text(&row, 2), + input: get_json(&row, 3), + output_raw: get_opt_text(&row, 4), + output_sanitized: get_opt_text(&row, 5).and_then(|s| serde_json::from_str(&s).ok()), + sanitization_warnings: warnings, + cost: get_opt_decimal(&row, 7), + duration: std::time::Duration::from_millis(get_i64(&row, 8) as u64), + success: get_i64(&row, 9) != 0, + error: get_opt_text(&row, 10), + executed_at: get_ts(&row, 11), + }); + } + Ok(actions) + } + + // ==================== LLM Calls ==================== + + async fn record_llm_call(&self, record: &LlmCallRecord<'_>) -> Result { + let conn = self.connect()?; + let id = Uuid::new_v4(); + conn.execute( + r#" + INSERT INTO llm_calls (id, job_id, conversation_id, provider, model, input_tokens, output_tokens, cost, purpose) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) + "#, + params![ + id.to_string(), + opt_text_owned(record.job_id.map(|id| id.to_string())), + opt_text_owned(record.conversation_id.map(|id| id.to_string())), + record.provider, + record.model, + record.input_tokens as i64, + record.output_tokens as i64, + record.cost.to_string(), + opt_text(record.purpose), + ], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(id) + } + + // ==================== Estimation Snapshots ==================== + + async fn save_estimation_snapshot( + &self, + job_id: Uuid, + category: &str, + tool_names: &[String], + estimated_cost: Decimal, + estimated_time_secs: i32, + estimated_value: Decimal, + ) -> Result { + let conn = self.connect()?; + let id = Uuid::new_v4(); + let tools_json = serde_json::to_string(tool_names) + .map_err(|e| DatabaseError::Serialization(e.to_string()))?; + + conn.execute( + r#" + INSERT INTO estimation_snapshots (id, job_id, category, tool_names, estimated_cost, estimated_time_secs, estimated_value) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + "#, + params![ + id.to_string(), + job_id.to_string(), + category, + tools_json, + estimated_cost.to_string(), + estimated_time_secs as i64, + estimated_value.to_string(), + ], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(id) + } + + async fn update_estimation_actuals( + &self, + id: Uuid, + actual_cost: Decimal, + actual_time_secs: i32, + actual_value: Option, + ) -> Result<(), DatabaseError> { + let conn = self.connect()?; + conn.execute( + "UPDATE estimation_snapshots SET actual_cost = ?2, actual_time_secs = ?3, actual_value = ?4 WHERE id = ?1", + params![ + id.to_string(), + actual_cost.to_string(), + actual_time_secs as i64, + actual_value.map(|d| d.to_string()).unwrap_or_default(), + ], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + // ==================== Sandbox Jobs ==================== + + async fn save_sandbox_job(&self, job: &SandboxJobRecord) -> Result<(), DatabaseError> { + let conn = self.connect()?; + conn.execute( + r#" + INSERT INTO agent_jobs ( + id, title, description, status, source, user_id, project_dir, + success, failure_reason, created_at, started_at, completed_at + ) VALUES (?1, ?2, '', ?3, 'sandbox', ?4, ?5, ?6, ?7, ?8, ?9, ?10) + ON CONFLICT (id) DO UPDATE SET + status = excluded.status, + success = excluded.success, + failure_reason = excluded.failure_reason, + started_at = excluded.started_at, + completed_at = excluded.completed_at + "#, + params![ + job.id.to_string(), + job.task.as_str(), + job.status.as_str(), + job.user_id.as_str(), + job.project_dir.as_str(), + job.success.map(|b| b as i64), + opt_text(job.failure_reason.as_deref()), + fmt_ts(&job.created_at), + fmt_opt_ts(&job.started_at), + fmt_opt_ts(&job.completed_at), + ], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn get_sandbox_job(&self, id: Uuid) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + r#" + SELECT id, title, status, user_id, project_dir, + success, failure_reason, created_at, started_at, completed_at + FROM agent_jobs WHERE id = ?1 AND source = 'sandbox' + "#, + params![id.to_string()], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + Some(row) => Ok(Some(SandboxJobRecord { + id: get_text(&row, 0).parse().unwrap_or_default(), + task: get_text(&row, 1), + status: get_text(&row, 2), + user_id: get_text(&row, 3), + project_dir: get_text(&row, 4), + success: get_opt_bool(&row, 5), + failure_reason: get_opt_text(&row, 6), + created_at: get_ts(&row, 7), + started_at: get_opt_ts(&row, 8), + completed_at: get_opt_ts(&row, 9), + })), + None => Ok(None), + } + } + + async fn list_sandbox_jobs(&self) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + r#" + SELECT id, title, status, user_id, project_dir, + success, failure_reason, created_at, started_at, completed_at + FROM agent_jobs WHERE source = 'sandbox' + ORDER BY created_at DESC + "#, + (), + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut jobs = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + jobs.push(SandboxJobRecord { + id: get_text(&row, 0).parse().unwrap_or_default(), + task: get_text(&row, 1), + status: get_text(&row, 2), + user_id: get_text(&row, 3), + project_dir: get_text(&row, 4), + success: get_opt_bool(&row, 5), + failure_reason: get_opt_text(&row, 6), + created_at: get_ts(&row, 7), + started_at: get_opt_ts(&row, 8), + completed_at: get_opt_ts(&row, 9), + }); + } + Ok(jobs) + } + + async fn update_sandbox_job_status( + &self, + id: Uuid, + status: &str, + success: Option, + message: Option<&str>, + started_at: Option>, + completed_at: Option>, + ) -> Result<(), DatabaseError> { + let conn = self.connect()?; + conn.execute( + r#" + UPDATE agent_jobs SET + status = ?2, + success = COALESCE(?3, success), + failure_reason = COALESCE(?4, failure_reason), + started_at = COALESCE(?5, started_at), + completed_at = COALESCE(?6, completed_at) + WHERE id = ?1 AND source = 'sandbox' + "#, + params![ + id.to_string(), + status, + success.map(|b| b as i64), + message, + fmt_opt_ts(&started_at), + fmt_opt_ts(&completed_at), + ], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn cleanup_stale_sandbox_jobs(&self) -> Result { + let conn = self.connect()?; + let now = fmt_ts(&Utc::now()); + let count = conn + .execute( + r#" + UPDATE agent_jobs SET + status = 'interrupted', + failure_reason = 'Process restarted', + completed_at = ?1 + WHERE source = 'sandbox' AND status IN ('running', 'creating') + "#, + params![now], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + if count > 0 { + tracing::info!("Marked {} stale sandbox jobs as interrupted", count); + } + Ok(count) + } + + async fn sandbox_job_summary(&self) -> Result { + let conn = self.connect()?; + let mut rows = conn + .query( + "SELECT status, COUNT(*) as cnt FROM agent_jobs WHERE source = 'sandbox' GROUP BY status", + (), + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut summary = SandboxJobSummary::default(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + let status = get_text(&row, 0); + let count = get_i64(&row, 1) as usize; + summary.total += count; + match status.as_str() { + "creating" => summary.creating += count, + "running" => summary.running += count, + "completed" => summary.completed += count, + "failed" => summary.failed += count, + "interrupted" => summary.interrupted += count, + _ => {} + } + } + Ok(summary) + } + + async fn list_sandbox_jobs_for_user( + &self, + user_id: &str, + ) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + r#" + SELECT id, title, status, user_id, project_dir, + success, failure_reason, created_at, started_at, completed_at + FROM agent_jobs WHERE source = 'sandbox' AND user_id = ?1 + ORDER BY created_at DESC + "#, + libsql::params![user_id], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut jobs = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + jobs.push(SandboxJobRecord { + id: get_text(&row, 0).parse().unwrap_or_default(), + task: get_text(&row, 1), + status: get_text(&row, 2), + user_id: get_text(&row, 3), + project_dir: get_text(&row, 4), + success: get_opt_bool(&row, 5), + failure_reason: get_opt_text(&row, 6), + created_at: get_ts(&row, 7), + started_at: get_opt_ts(&row, 8), + completed_at: get_opt_ts(&row, 9), + }); + } + Ok(jobs) + } + + async fn sandbox_job_summary_for_user( + &self, + user_id: &str, + ) -> Result { + let conn = self.connect()?; + let mut rows = conn + .query( + "SELECT status, COUNT(*) as cnt FROM agent_jobs WHERE source = 'sandbox' AND user_id = ?1 GROUP BY status", + libsql::params![user_id], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut summary = SandboxJobSummary::default(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + let status = get_text(&row, 0); + let count = get_i64(&row, 1) as usize; + summary.total += count; + match status.as_str() { + "creating" => summary.creating += count, + "running" => summary.running += count, + "completed" => summary.completed += count, + "failed" => summary.failed += count, + "interrupted" => summary.interrupted += count, + _ => {} + } + } + Ok(summary) + } + + async fn sandbox_job_belongs_to_user( + &self, + job_id: Uuid, + user_id: &str, + ) -> Result { + let conn = self.connect()?; + let mut rows = conn + .query( + "SELECT 1 FROM agent_jobs WHERE id = ?1 AND user_id = ?2 AND source = 'sandbox'", + libsql::params![job_id.to_string(), user_id], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + let found = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(found.is_some()) + } + + async fn update_sandbox_job_mode(&self, id: Uuid, mode: &str) -> Result<(), DatabaseError> { + let conn = self.connect()?; + conn.execute( + "UPDATE agent_jobs SET job_mode = ?2 WHERE id = ?1", + params![id.to_string(), mode], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn get_sandbox_job_mode(&self, id: Uuid) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + "SELECT job_mode FROM agent_jobs WHERE id = ?1", + params![id.to_string()], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + Some(row) => Ok(Some(get_text(&row, 0))), + None => Ok(None), + } + } + + // ==================== Job Events ==================== + + async fn save_job_event( + &self, + job_id: Uuid, + event_type: &str, + data: &serde_json::Value, + ) -> Result<(), DatabaseError> { + let conn = self.connect()?; + conn.execute( + "INSERT INTO job_events (job_id, event_type, data) VALUES (?1, ?2, ?3)", + params![job_id.to_string(), event_type, data.to_string()], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn list_job_events(&self, job_id: Uuid) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + r#" + SELECT id, job_id, event_type, data, created_at + FROM job_events WHERE job_id = ?1 ORDER BY id ASC + "#, + params![job_id.to_string()], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut events = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + events.push(JobEventRecord { + id: get_i64(&row, 0), + job_id: get_text(&row, 1).parse().unwrap_or_default(), + event_type: get_text(&row, 2), + data: get_json(&row, 3), + created_at: get_ts(&row, 4), + }); + } + Ok(events) + } + + // ==================== Routines ==================== + + async fn create_routine(&self, routine: &Routine) -> Result<(), DatabaseError> { + let conn = self.connect()?; + let trigger_type = routine.trigger.type_tag(); + let trigger_config = routine.trigger.to_config_json(); + let action_type = routine.action.type_tag(); + let action_config = routine.action.to_config_json(); + let cooldown_secs = routine.guardrails.cooldown.as_secs() as i64; + let max_concurrent = routine.guardrails.max_concurrent as i64; + let dedup_window_secs = routine.guardrails.dedup_window.map(|d| d.as_secs() as i64); + + conn.execute( + r#" + INSERT INTO routines ( + id, name, description, user_id, enabled, + trigger_type, trigger_config, action_type, action_config, + cooldown_secs, max_concurrent, dedup_window_secs, + notify_channel, notify_user, notify_on_success, notify_on_failure, notify_on_attention, + state, next_fire_at, created_at, updated_at + ) VALUES ( + ?1, ?2, ?3, ?4, ?5, + ?6, ?7, ?8, ?9, + ?10, ?11, ?12, + ?13, ?14, ?15, ?16, ?17, + ?18, ?19, ?20, ?21 + ) + "#, + params![ + routine.id.to_string(), + routine.name.as_str(), + routine.description.as_str(), + routine.user_id.as_str(), + routine.enabled as i64, + trigger_type, + trigger_config.to_string(), + action_type, + action_config.to_string(), + cooldown_secs, + max_concurrent, + dedup_window_secs, + opt_text(routine.notify.channel.as_deref()), + routine.notify.user.as_str(), + routine.notify.on_success as i64, + routine.notify.on_failure as i64, + routine.notify.on_attention as i64, + routine.state.to_string(), + fmt_opt_ts(&routine.next_fire_at), + fmt_ts(&routine.created_at), + fmt_ts(&routine.updated_at), + ], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn get_routine(&self, id: Uuid) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + &format!("SELECT {} FROM routines WHERE id = ?1", ROUTINE_COLUMNS), + params![id.to_string()], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + Some(row) => Ok(Some(row_to_routine_libsql(&row)?)), + None => Ok(None), + } + } + + async fn get_routine_by_name( + &self, + user_id: &str, + name: &str, + ) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + &format!( + "SELECT {} FROM routines WHERE user_id = ?1 AND name = ?2", + ROUTINE_COLUMNS + ), + params![user_id, name], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + Some(row) => Ok(Some(row_to_routine_libsql(&row)?)), + None => Ok(None), + } + } + + async fn list_routines(&self, user_id: &str) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + &format!( + "SELECT {} FROM routines WHERE user_id = ?1 ORDER BY name", + ROUTINE_COLUMNS + ), + params![user_id], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut routines = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + routines.push(row_to_routine_libsql(&row)?); + } + Ok(routines) + } + + async fn list_event_routines(&self) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + &format!( + "SELECT {} FROM routines WHERE enabled = 1 AND trigger_type = 'event'", + ROUTINE_COLUMNS + ), + (), + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut routines = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + routines.push(row_to_routine_libsql(&row)?); + } + Ok(routines) + } + + async fn list_due_cron_routines(&self) -> Result, DatabaseError> { + let conn = self.connect()?; + let now = fmt_ts(&Utc::now()); + let mut rows = conn + .query( + &format!( + "SELECT {} FROM routines WHERE enabled = 1 AND trigger_type = 'cron' AND next_fire_at IS NOT NULL AND next_fire_at <= ?1", + ROUTINE_COLUMNS + ), + params![now], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut routines = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + routines.push(row_to_routine_libsql(&row)?); + } + Ok(routines) + } + + async fn update_routine(&self, routine: &Routine) -> Result<(), DatabaseError> { + let conn = self.connect()?; + let trigger_type = routine.trigger.type_tag(); + let trigger_config = routine.trigger.to_config_json(); + let action_type = routine.action.type_tag(); + let action_config = routine.action.to_config_json(); + let cooldown_secs = routine.guardrails.cooldown.as_secs() as i64; + let max_concurrent = routine.guardrails.max_concurrent as i64; + let dedup_window_secs = routine.guardrails.dedup_window.map(|d| d.as_secs() as i64); + let now = fmt_ts(&Utc::now()); + + conn.execute( + r#" + UPDATE routines SET + name = ?2, description = ?3, enabled = ?4, + trigger_type = ?5, trigger_config = ?6, + action_type = ?7, action_config = ?8, + cooldown_secs = ?9, max_concurrent = ?10, dedup_window_secs = ?11, + notify_channel = ?12, notify_user = ?13, + notify_on_success = ?14, notify_on_failure = ?15, notify_on_attention = ?16, + state = ?17, next_fire_at = ?18, + updated_at = ?19 + WHERE id = ?1 + "#, + params![ + routine.id.to_string(), + routine.name.as_str(), + routine.description.as_str(), + routine.enabled as i64, + trigger_type, + trigger_config.to_string(), + action_type, + action_config.to_string(), + cooldown_secs, + max_concurrent, + dedup_window_secs, + opt_text(routine.notify.channel.as_deref()), + routine.notify.user.as_str(), + routine.notify.on_success as i64, + routine.notify.on_failure as i64, + routine.notify.on_attention as i64, + routine.state.to_string(), + fmt_opt_ts(&routine.next_fire_at), + now, + ], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn update_routine_runtime( + &self, + id: Uuid, + last_run_at: DateTime, + next_fire_at: Option>, + run_count: u64, + consecutive_failures: u32, + state: &serde_json::Value, + ) -> Result<(), DatabaseError> { + let conn = self.connect()?; + let now = fmt_ts(&Utc::now()); + conn.execute( + r#" + UPDATE routines SET + last_run_at = ?2, next_fire_at = ?3, + run_count = ?4, consecutive_failures = ?5, + state = ?6, updated_at = ?7 + WHERE id = ?1 + "#, + params![ + id.to_string(), + fmt_ts(&last_run_at), + fmt_opt_ts(&next_fire_at), + run_count as i64, + consecutive_failures as i64, + state.to_string(), + now, + ], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn delete_routine(&self, id: Uuid) -> Result { + let conn = self.connect()?; + let count = conn + .execute( + "DELETE FROM routines WHERE id = ?1", + params![id.to_string()], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(count > 0) + } + + // ==================== Routine Runs ==================== + + async fn create_routine_run(&self, run: &RoutineRun) -> Result<(), DatabaseError> { + let conn = self.connect()?; + conn.execute( + r#" + INSERT INTO routine_runs ( + id, routine_id, trigger_type, trigger_detail, + started_at, status, job_id + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7) + "#, + params![ + run.id.to_string(), + run.routine_id.to_string(), + run.trigger_type.as_str(), + opt_text(run.trigger_detail.as_deref()), + fmt_ts(&run.started_at), + run.status.to_string(), + opt_text_owned(run.job_id.map(|id| id.to_string())), + ], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn complete_routine_run( + &self, + id: Uuid, + status: RunStatus, + result_summary: Option<&str>, + tokens_used: Option, + ) -> Result<(), DatabaseError> { + let conn = self.connect()?; + let now = fmt_ts(&Utc::now()); + conn.execute( + r#" + UPDATE routine_runs SET + completed_at = ?5, status = ?2, + result_summary = ?3, tokens_used = ?4 + WHERE id = ?1 + "#, + params![ + id.to_string(), + status.to_string(), + opt_text(result_summary), + tokens_used.map(|t| t as i64), + now, + ], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn list_routine_runs( + &self, + routine_id: Uuid, + limit: i64, + ) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + &format!( + "SELECT {} FROM routine_runs WHERE routine_id = ?1 ORDER BY started_at DESC LIMIT ?2", + ROUTINE_RUN_COLUMNS + ), + params![routine_id.to_string(), limit], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut runs = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + runs.push(row_to_routine_run_libsql(&row)?); + } + Ok(runs) + } + + async fn count_running_routine_runs(&self, routine_id: Uuid) -> Result { + let conn = self.connect()?; + let mut rows = conn + .query( + "SELECT COUNT(*) as cnt FROM routine_runs WHERE routine_id = ?1 AND status = 'running'", + params![routine_id.to_string()], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + Some(row) => Ok(get_i64(&row, 0)), + None => Ok(0), + } + } + + // ==================== Tool Failures ==================== + + async fn record_tool_failure( + &self, + tool_name: &str, + error_message: &str, + ) -> Result<(), DatabaseError> { + let conn = self.connect()?; + let now = fmt_ts(&Utc::now()); + conn.execute( + r#" + INSERT INTO tool_failures (id, tool_name, error_message, error_count, last_failure) + VALUES (?1, ?2, ?3, 1, ?4) + ON CONFLICT (tool_name) DO UPDATE SET + error_message = ?3, + error_count = tool_failures.error_count + 1, + last_failure = ?4 + "#, + params![Uuid::new_v4().to_string(), tool_name, error_message, now], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn get_broken_tools(&self, threshold: i32) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + r#" + SELECT tool_name, error_message, error_count, first_failure, last_failure, + last_build_result, repair_attempts + FROM tool_failures + WHERE error_count >= ?1 AND repaired_at IS NULL + ORDER BY error_count DESC + "#, + params![threshold as i64], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut tools = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + tools.push(BrokenTool { + name: get_text(&row, 0), + last_error: get_opt_text(&row, 1), + failure_count: get_i64(&row, 2) as u32, + first_failure: get_ts(&row, 3), + last_failure: get_ts(&row, 4), + last_build_result: get_opt_text(&row, 5) + .and_then(|s| serde_json::from_str(&s).ok()), + repair_attempts: get_i64(&row, 6) as u32, + }); + } + Ok(tools) + } + + async fn mark_tool_repaired(&self, tool_name: &str) -> Result<(), DatabaseError> { + let conn = self.connect()?; + let now = fmt_ts(&Utc::now()); + conn.execute( + "UPDATE tool_failures SET repaired_at = ?2, error_count = 0 WHERE tool_name = ?1", + params![tool_name, now], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn increment_repair_attempts(&self, tool_name: &str) -> Result<(), DatabaseError> { + let conn = self.connect()?; + conn.execute( + "UPDATE tool_failures SET repair_attempts = repair_attempts + 1 WHERE tool_name = ?1", + params![tool_name], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + // ==================== Settings ==================== + + async fn get_setting( + &self, + user_id: &str, + key: &str, + ) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + "SELECT value FROM settings WHERE user_id = ?1 AND key = ?2", + params![user_id, key], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + Some(row) => Ok(Some(get_json(&row, 0))), + None => Ok(None), + } + } + + async fn get_setting_full( + &self, + user_id: &str, + key: &str, + ) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + "SELECT key, value, updated_at FROM settings WHERE user_id = ?1 AND key = ?2", + params![user_id, key], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + Some(row) => Ok(Some(SettingRow { + key: get_text(&row, 0), + value: get_json(&row, 1), + updated_at: get_ts(&row, 2), + })), + None => Ok(None), + } + } + + async fn set_setting( + &self, + user_id: &str, + key: &str, + value: &serde_json::Value, + ) -> Result<(), DatabaseError> { + let conn = self.connect()?; + let now = fmt_ts(&Utc::now()); + conn.execute( + r#" + INSERT INTO settings (user_id, key, value, updated_at) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT (user_id, key) DO UPDATE SET + value = excluded.value, + updated_at = ?4 + "#, + params![user_id, key, value.to_string(), now], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn delete_setting(&self, user_id: &str, key: &str) -> Result { + let conn = self.connect()?; + let count = conn + .execute( + "DELETE FROM settings WHERE user_id = ?1 AND key = ?2", + params![user_id, key], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(count > 0) + } + + async fn list_settings(&self, user_id: &str) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + "SELECT key, value, updated_at FROM settings WHERE user_id = ?1 ORDER BY key", + params![user_id], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut settings = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + settings.push(SettingRow { + key: get_text(&row, 0), + value: get_json(&row, 1), + updated_at: get_ts(&row, 2), + }); + } + Ok(settings) + } + + async fn get_all_settings( + &self, + user_id: &str, + ) -> Result, DatabaseError> { + let conn = self.connect()?; + let mut rows = conn + .query( + "SELECT key, value FROM settings WHERE user_id = ?1", + params![user_id], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + let mut map = HashMap::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + map.insert(get_text(&row, 0), get_json(&row, 1)); + } + Ok(map) + } + + async fn set_all_settings( + &self, + user_id: &str, + settings: &HashMap, + ) -> Result<(), DatabaseError> { + let conn = self.connect()?; + let now = fmt_ts(&Utc::now()); + conn.execute("BEGIN", ()) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + for (key, value) in settings { + if let Err(e) = conn + .execute( + r#" + INSERT INTO settings (user_id, key, value, updated_at) + VALUES (?1, ?2, ?3, ?4) + ON CONFLICT (user_id, key) DO UPDATE SET + value = excluded.value, + updated_at = ?4 + "#, + params![user_id, key.as_str(), value.to_string(), now.as_str()], + ) + .await + { + let _ = conn.execute("ROLLBACK", ()).await; + return Err(DatabaseError::Query(e.to_string())); + } + } + + conn.execute("COMMIT", ()) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + Ok(()) + } + + async fn has_settings(&self, user_id: &str) -> Result { + let conn = self.connect()?; + let mut rows = conn + .query( + "SELECT COUNT(*) as cnt FROM settings WHERE user_id = ?1", + params![user_id], + ) + .await + .map_err(|e| DatabaseError::Query(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| DatabaseError::Query(e.to_string()))? + { + Some(row) => Ok(get_i64(&row, 0) > 0), + None => Ok(false), + } + } + + // ==================== Workspace: Documents ==================== + + async fn get_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result { + let conn = self.connect().map_err(|e| WorkspaceError::SearchFailed { + reason: e.to_string(), + })?; + let agent_id_str = agent_id.map(|id| id.to_string()); + let mut rows = conn + .query( + r#" + SELECT id, user_id, agent_id, path, content, + created_at, updated_at, metadata + FROM memory_documents + WHERE user_id = ?1 AND agent_id IS ?2 AND path = ?3 + "#, + params![user_id, agent_id_str.as_deref(), path], + ) + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Query failed: {}", e), + })?; + + match rows + .next() + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Query failed: {}", e), + })? { + Some(row) => Ok(row_to_memory_document(&row)), + None => Err(WorkspaceError::DocumentNotFound { + doc_type: path.to_string(), + user_id: user_id.to_string(), + }), + } + } + + async fn get_document_by_id(&self, id: Uuid) -> Result { + let conn = self.connect().map_err(|e| WorkspaceError::SearchFailed { + reason: e.to_string(), + })?; + let mut rows = conn + .query( + r#" + SELECT id, user_id, agent_id, path, content, + created_at, updated_at, metadata + FROM memory_documents WHERE id = ?1 + "#, + params![id.to_string()], + ) + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Query failed: {}", e), + })?; + + match rows + .next() + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Query failed: {}", e), + })? { + Some(row) => Ok(row_to_memory_document(&row)), + None => Err(WorkspaceError::DocumentNotFound { + doc_type: "unknown".to_string(), + user_id: "unknown".to_string(), + }), + } + } + + async fn get_or_create_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result { + // Try get + match self.get_document_by_path(user_id, agent_id, path).await { + Ok(doc) => return Ok(doc), + Err(WorkspaceError::DocumentNotFound { .. }) => {} + Err(e) => return Err(e), + } + + // Create + let conn = self.connect().map_err(|e| WorkspaceError::SearchFailed { + reason: e.to_string(), + })?; + let id = Uuid::new_v4(); + let agent_id_str = agent_id.map(|id| id.to_string()); + conn.execute( + r#" + INSERT INTO memory_documents (id, user_id, agent_id, path, content, metadata) + VALUES (?1, ?2, ?3, ?4, '', '{}') + ON CONFLICT (user_id, agent_id, path) DO NOTHING + "#, + params![id.to_string(), user_id, agent_id_str.as_deref(), path], + ) + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Insert failed: {}", e), + })?; + + self.get_document_by_path(user_id, agent_id, path).await + } + + async fn update_document(&self, id: Uuid, content: &str) -> Result<(), WorkspaceError> { + let conn = self.connect().map_err(|e| WorkspaceError::SearchFailed { + reason: e.to_string(), + })?; + let now = fmt_ts(&Utc::now()); + conn.execute( + "UPDATE memory_documents SET content = ?2, updated_at = ?3 WHERE id = ?1", + params![id.to_string(), content, now], + ) + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Update failed: {}", e), + })?; + Ok(()) + } + + async fn delete_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result<(), WorkspaceError> { + let doc = self.get_document_by_path(user_id, agent_id, path).await?; + self.delete_chunks(doc.id).await?; + + let conn = self.connect().map_err(|e| WorkspaceError::SearchFailed { + reason: e.to_string(), + })?; + let agent_id_str = agent_id.map(|id| id.to_string()); + conn.execute( + "DELETE FROM memory_documents WHERE user_id = ?1 AND agent_id IS ?2 AND path = ?3", + params![user_id, agent_id_str.as_deref(), path], + ) + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Delete failed: {}", e), + })?; + Ok(()) + } + + async fn list_directory( + &self, + user_id: &str, + agent_id: Option, + directory: &str, + ) -> Result, WorkspaceError> { + let conn = self.connect().map_err(|e| WorkspaceError::SearchFailed { + reason: e.to_string(), + })?; + // Implement the list_workspace_files logic in Rust instead of PL/pgSQL. + let dir = if !directory.is_empty() && !directory.ends_with('/') { + format!("{}/", directory) + } else { + directory.to_string() + }; + + let agent_id_str = agent_id.map(|id| id.to_string()); + let pattern = if dir.is_empty() { + "%".to_string() + } else { + format!("{}%", dir) + }; + + let mut rows = conn + .query( + r#" + SELECT path, updated_at, substr(content, 1, 200) as content_preview + FROM memory_documents + WHERE user_id = ?1 AND agent_id IS ?2 + AND (?3 = '%' OR path LIKE ?3) + ORDER BY path + "#, + params![user_id, agent_id_str.as_deref(), pattern], + ) + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("List directory failed: {}", e), + })?; + + let mut entries_map: HashMap = HashMap::new(); + + while let Some(row) = rows + .next() + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Query failed: {}", e), + })? + { + let full_path = get_text(&row, 0); + let updated_at = get_opt_ts(&row, 1); + let content_preview = get_opt_text(&row, 2); + + // Extract the immediate child name relative to directory + let relative = if dir.is_empty() { + &full_path + } else if let Some(stripped) = full_path.strip_prefix(&dir) { + stripped + } else { + continue; + }; + + let child_name = if let Some(slash_pos) = relative.find('/') { + &relative[..slash_pos] + } else { + relative + }; + + if child_name.is_empty() { + continue; + } + + let is_dir = relative.contains('/'); + let entry_path = if dir.is_empty() { + child_name.to_string() + } else { + format!("{}{}", dir, child_name) + }; + + entries_map + .entry(child_name.to_string()) + .and_modify(|e| { + // Mark as directory if any sub-paths exist + if is_dir { + e.is_directory = true; + e.content_preview = None; + } + // Update to latest timestamp + if let (Some(existing), Some(new)) = (&e.updated_at, &updated_at) + && new > existing + { + e.updated_at = Some(*new); + } + }) + .or_insert(WorkspaceEntry { + path: entry_path, + is_directory: is_dir, + updated_at, + content_preview: if is_dir { None } else { content_preview }, + }); + } + + let mut entries: Vec = entries_map.into_values().collect(); + entries.sort_by(|a, b| a.path.cmp(&b.path)); + Ok(entries) + } + + async fn list_all_paths( + &self, + user_id: &str, + agent_id: Option, + ) -> Result, WorkspaceError> { + let conn = self.connect().map_err(|e| WorkspaceError::SearchFailed { + reason: e.to_string(), + })?; + let agent_id_str = agent_id.map(|id| id.to_string()); + let mut rows = conn + .query( + "SELECT path FROM memory_documents WHERE user_id = ?1 AND agent_id IS ?2 ORDER BY path", + params![user_id, agent_id_str.as_deref()], + ) + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("List paths failed: {}", e), + })?; + + let mut paths = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Query failed: {}", e), + })? + { + paths.push(get_text(&row, 0)); + } + Ok(paths) + } + + async fn list_documents( + &self, + user_id: &str, + agent_id: Option, + ) -> Result, WorkspaceError> { + let conn = self.connect().map_err(|e| WorkspaceError::SearchFailed { + reason: e.to_string(), + })?; + let agent_id_str = agent_id.map(|id| id.to_string()); + let mut rows = conn + .query( + r#" + SELECT id, user_id, agent_id, path, content, + created_at, updated_at, metadata + FROM memory_documents + WHERE user_id = ?1 AND agent_id IS ?2 + ORDER BY updated_at DESC + "#, + params![user_id, agent_id_str.as_deref()], + ) + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Query failed: {}", e), + })?; + + let mut docs = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Query failed: {}", e), + })? + { + docs.push(row_to_memory_document(&row)); + } + Ok(docs) + } + + // ==================== Workspace: Chunks ==================== + + async fn delete_chunks(&self, document_id: Uuid) -> Result<(), WorkspaceError> { + let conn = self.connect().map_err(|e| WorkspaceError::ChunkingFailed { + reason: e.to_string(), + })?; + conn.execute( + "DELETE FROM memory_chunks WHERE document_id = ?1", + params![document_id.to_string()], + ) + .await + .map_err(|e| WorkspaceError::ChunkingFailed { + reason: format!("Delete failed: {}", e), + })?; + Ok(()) + } + + async fn insert_chunk( + &self, + document_id: Uuid, + chunk_index: i32, + content: &str, + embedding: Option<&[f32]>, + ) -> Result { + let conn = self.connect().map_err(|e| WorkspaceError::ChunkingFailed { + reason: e.to_string(), + })?; + let id = Uuid::new_v4(); + let embedding_blob = embedding.map(|e| { + // Convert f32 slice to bytes for F32_BLOB + let bytes: Vec = e.iter().flat_map(|f| f.to_le_bytes()).collect(); + bytes + }); + + conn.execute( + r#" + INSERT INTO memory_chunks (id, document_id, chunk_index, content, embedding) + VALUES (?1, ?2, ?3, ?4, ?5) + "#, + params![ + id.to_string(), + document_id.to_string(), + chunk_index as i64, + content, + embedding_blob.map(libsql::Value::Blob), + ], + ) + .await + .map_err(|e| WorkspaceError::ChunkingFailed { + reason: format!("Insert failed: {}", e), + })?; + Ok(id) + } + + async fn update_chunk_embedding( + &self, + chunk_id: Uuid, + embedding: &[f32], + ) -> Result<(), WorkspaceError> { + let conn = self + .connect() + .map_err(|e| WorkspaceError::EmbeddingFailed { + reason: e.to_string(), + })?; + let bytes: Vec = embedding.iter().flat_map(|f| f.to_le_bytes()).collect(); + + conn.execute( + "UPDATE memory_chunks SET embedding = ?2 WHERE id = ?1", + params![chunk_id.to_string(), libsql::Value::Blob(bytes)], + ) + .await + .map_err(|e| WorkspaceError::EmbeddingFailed { + reason: format!("Update failed: {}", e), + })?; + Ok(()) + } + + async fn get_chunks_without_embeddings( + &self, + user_id: &str, + agent_id: Option, + limit: usize, + ) -> Result, WorkspaceError> { + let conn = self.connect().map_err(|e| WorkspaceError::SearchFailed { + reason: e.to_string(), + })?; + let agent_id_str = agent_id.map(|id| id.to_string()); + let mut rows = conn + .query( + r#" + SELECT c.id, c.document_id, c.chunk_index, c.content, c.created_at + FROM memory_chunks c + JOIN memory_documents d ON d.id = c.document_id + WHERE d.user_id = ?1 AND d.agent_id IS ?2 + AND c.embedding IS NULL + LIMIT ?3 + "#, + params![user_id, agent_id_str.as_deref(), limit as i64], + ) + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Query failed: {}", e), + })?; + + let mut chunks = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Query failed: {}", e), + })? + { + chunks.push(MemoryChunk { + id: get_text(&row, 0).parse().unwrap_or_default(), + document_id: get_text(&row, 1).parse().unwrap_or_default(), + chunk_index: get_i64(&row, 2) as i32, + content: get_text(&row, 3), + embedding: None, + created_at: get_ts(&row, 4), + }); + } + Ok(chunks) + } + + // ==================== Workspace: Search ==================== + + async fn hybrid_search( + &self, + user_id: &str, + agent_id: Option, + query: &str, + embedding: Option<&[f32]>, + config: &SearchConfig, + ) -> Result, WorkspaceError> { + let conn = self.connect().map_err(|e| WorkspaceError::SearchFailed { + reason: e.to_string(), + })?; + let agent_id_str = agent_id.map(|id| id.to_string()); + let pre_limit = config.pre_fusion_limit as i64; + + // FTS search using FTS5 + let fts_results = if config.use_fts { + let mut rows = conn + .query( + r#" + SELECT c.id, c.document_id, c.content + FROM memory_chunks_fts fts + JOIN memory_chunks c ON c._rowid = fts.rowid + JOIN memory_documents d ON d.id = c.document_id + WHERE d.user_id = ?1 AND d.agent_id IS ?2 + AND memory_chunks_fts MATCH ?3 + ORDER BY rank + LIMIT ?4 + "#, + params![user_id, agent_id_str.as_deref(), query, pre_limit], + ) + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("FTS query failed: {}", e), + })?; + + let mut results = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("FTS row fetch failed: {}", e), + })? + { + results.push(RankedResult { + chunk_id: get_text(&row, 0).parse().unwrap_or_default(), + document_id: get_text(&row, 1).parse().unwrap_or_default(), + content: get_text(&row, 2), + rank: results.len() as u32 + 1, + }); + } + results + } else { + Vec::new() + }; + + // Vector search using libsql_vector_idx + let vector_results = if let (true, Some(emb)) = (config.use_vector, embedding) { + // Format as JSON array string for vector() SQL function + let vector_json = format!( + "[{}]", + emb.iter() + .map(|f| f.to_string()) + .collect::>() + .join(",") + ); + + // vector_top_k returns rowids from the vector index. + // We join back to memory_chunks and filter by user/agent. + let mut rows = conn + .query( + r#" + SELECT c.id, c.document_id, c.content + FROM vector_top_k('idx_memory_chunks_embedding', vector(?1), ?2) AS top_k + JOIN memory_chunks c ON c._rowid = top_k.id + JOIN memory_documents d ON d.id = c.document_id + WHERE d.user_id = ?3 AND d.agent_id IS ?4 + "#, + params![vector_json, pre_limit, user_id, agent_id_str.as_deref()], + ) + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Vector query failed: {}", e), + })?; + + let mut results = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Vector row fetch failed: {}", e), + })? + { + results.push(RankedResult { + chunk_id: get_text(&row, 0).parse().unwrap_or_default(), + document_id: get_text(&row, 1).parse().unwrap_or_default(), + content: get_text(&row, 2), + rank: results.len() as u32 + 1, + }); + } + results + } else { + Vec::new() + }; + + if embedding.is_some() && !config.use_vector { + tracing::warn!( + "Embedding provided but vector search is disabled in config; using FTS-only results" + ); + } + + Ok(reciprocal_rank_fusion(fts_results, vector_results, config)) + } +} + +// ==================== Row conversion helpers ==================== + +fn row_to_memory_document(row: &libsql::Row) -> MemoryDocument { + MemoryDocument { + id: get_text(row, 0).parse().unwrap_or_default(), + user_id: get_text(row, 1), + agent_id: get_opt_text(row, 2).and_then(|s| s.parse().ok()), + path: get_text(row, 3), + content: get_text(row, 4), + created_at: get_ts(row, 5), + updated_at: get_ts(row, 6), + metadata: get_json(row, 7), + } +} + +fn row_to_routine_libsql(row: &libsql::Row) -> Result { + let trigger_type = get_text(row, 5); + let trigger_config = get_json(row, 6); + let action_type = get_text(row, 7); + let action_config = get_json(row, 8); + let cooldown_secs = get_i64(row, 9); + let max_concurrent = get_i64(row, 10); + let dedup_window_secs: Option = row.get::(11).ok(); + + let trigger = + Trigger::from_db(&trigger_type, trigger_config).map_err(DatabaseError::Serialization)?; + let action = RoutineAction::from_db(&action_type, action_config) + .map_err(DatabaseError::Serialization)?; + + Ok(Routine { + id: get_text(row, 0).parse().unwrap_or_default(), + name: get_text(row, 1), + description: get_text(row, 2), + user_id: get_text(row, 3), + enabled: get_i64(row, 4) != 0, + trigger, + action, + guardrails: RoutineGuardrails { + cooldown: std::time::Duration::from_secs(cooldown_secs as u64), + max_concurrent: max_concurrent as u32, + dedup_window: dedup_window_secs.map(|s| std::time::Duration::from_secs(s as u64)), + }, + notify: NotifyConfig { + channel: get_opt_text(row, 12), + user: get_text(row, 13), + on_success: get_i64(row, 14) != 0, + on_failure: get_i64(row, 15) != 0, + on_attention: get_i64(row, 16) != 0, + }, + state: get_json(row, 17), + last_run_at: get_opt_ts(row, 18), + next_fire_at: get_opt_ts(row, 19), + run_count: get_i64(row, 20) as u64, + consecutive_failures: get_i64(row, 21) as u32, + created_at: get_ts(row, 22), + updated_at: get_ts(row, 23), + }) +} + +fn row_to_routine_run_libsql(row: &libsql::Row) -> Result { + let status_str = get_text(row, 5); + let status: RunStatus = status_str + .parse() + .map_err(|e: String| DatabaseError::Serialization(e))?; + + Ok(RoutineRun { + id: get_text(row, 0).parse().unwrap_or_default(), + routine_id: get_text(row, 1).parse().unwrap_or_default(), + trigger_type: get_text(row, 2), + trigger_detail: get_opt_text(row, 3), + started_at: get_ts(row, 4), + completed_at: get_opt_ts(row, 6), + status, + result_summary: get_opt_text(row, 7), + tokens_used: row.get::(8).ok().map(|v| v as i32), + job_id: get_opt_text(row, 9).and_then(|s| s.parse().ok()), + created_at: get_ts(row, 10), + }) +} diff --git a/src/db/libsql_migrations.rs b/src/db/libsql_migrations.rs new file mode 100644 index 00000000..1480ed7d --- /dev/null +++ b/src/db/libsql_migrations.rs @@ -0,0 +1,549 @@ +//! SQLite-dialect migrations for the libSQL/Turso backend. +//! +//! Consolidates all PostgreSQL migrations (V1-V8) into a single SQLite-compatible +//! schema. Run once on database creation; idempotent via `IF NOT EXISTS`. + +/// Consolidated schema for libSQL. +/// +/// Translates PostgreSQL types and features: +/// - `UUID` -> `TEXT` (store as hex string) +/// - `TIMESTAMPTZ` -> `TEXT` (ISO-8601) +/// - `JSONB` -> `TEXT` (JSON encoded) +/// - `BYTEA` -> `BLOB` +/// - `NUMERIC` -> `TEXT` (preserve precision for rust_decimal) +/// - `TEXT[]` -> `TEXT` (JSON array) +/// - `VECTOR(1536)` -> `F32_BLOB(1536)` (libsql native) +/// - `TSVECTOR` -> FTS5 virtual table +/// - `BIGSERIAL` -> `INTEGER PRIMARY KEY AUTOINCREMENT` +/// - PL/pgSQL functions -> SQLite triggers +pub const SCHEMA: &str = r#" + +-- ==================== Migration tracking ==================== + +CREATE TABLE IF NOT EXISTS _migrations ( + version INTEGER PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- ==================== Conversations ==================== + +CREATE TABLE IF NOT EXISTS conversations ( + id TEXT PRIMARY KEY, + channel TEXT NOT NULL, + user_id TEXT NOT NULL, + thread_id TEXT, + started_at TEXT NOT NULL DEFAULT (datetime('now')), + last_activity TEXT NOT NULL DEFAULT (datetime('now')), + metadata TEXT NOT NULL DEFAULT '{}' +); + +CREATE INDEX IF NOT EXISTS idx_conversations_channel ON conversations(channel); +CREATE INDEX IF NOT EXISTS idx_conversations_user ON conversations(user_id); +CREATE INDEX IF NOT EXISTS idx_conversations_last_activity ON conversations(last_activity); + +CREATE TABLE IF NOT EXISTS conversation_messages ( + id TEXT PRIMARY KEY, + conversation_id TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE, + role TEXT NOT NULL, + content TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_conversation_messages_conversation + ON conversation_messages(conversation_id); + +-- ==================== Agent Jobs ==================== + +CREATE TABLE IF NOT EXISTS agent_jobs ( + id TEXT PRIMARY KEY, + marketplace_job_id TEXT, + conversation_id TEXT REFERENCES conversations(id), + title TEXT NOT NULL, + description TEXT NOT NULL, + category TEXT, + status TEXT NOT NULL, + source TEXT NOT NULL, + user_id TEXT NOT NULL DEFAULT 'default', + project_dir TEXT, + job_mode TEXT NOT NULL DEFAULT 'worker', + budget_amount TEXT, + budget_token TEXT, + bid_amount TEXT, + estimated_cost TEXT, + estimated_time_secs INTEGER, + estimated_value TEXT, + actual_cost TEXT, + actual_time_secs INTEGER, + success INTEGER, + failure_reason TEXT, + stuck_since TEXT, + repair_attempts INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + started_at TEXT, + completed_at TEXT +); + +CREATE INDEX IF NOT EXISTS idx_agent_jobs_status ON agent_jobs(status); +CREATE INDEX IF NOT EXISTS idx_agent_jobs_marketplace ON agent_jobs(marketplace_job_id); +CREATE INDEX IF NOT EXISTS idx_agent_jobs_conversation ON agent_jobs(conversation_id); +CREATE INDEX IF NOT EXISTS idx_agent_jobs_source ON agent_jobs(source); +CREATE INDEX IF NOT EXISTS idx_agent_jobs_user ON agent_jobs(user_id); +CREATE INDEX IF NOT EXISTS idx_agent_jobs_created ON agent_jobs(created_at DESC); + +CREATE TABLE IF NOT EXISTS job_actions ( + id TEXT PRIMARY KEY, + job_id TEXT NOT NULL REFERENCES agent_jobs(id) ON DELETE CASCADE, + sequence_num INTEGER NOT NULL, + tool_name TEXT NOT NULL, + input TEXT NOT NULL, + output_raw TEXT, + output_sanitized TEXT, + sanitization_warnings TEXT, + cost TEXT, + duration_ms INTEGER, + success INTEGER NOT NULL, + error_message TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(job_id, sequence_num) +); + +CREATE INDEX IF NOT EXISTS idx_job_actions_job_id ON job_actions(job_id); +CREATE INDEX IF NOT EXISTS idx_job_actions_tool ON job_actions(tool_name); + +-- ==================== Dynamic Tools ==================== + +CREATE TABLE IF NOT EXISTS dynamic_tools ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + description TEXT NOT NULL, + parameters_schema TEXT NOT NULL, + code TEXT NOT NULL, + sandbox_config TEXT NOT NULL, + created_by_job_id TEXT REFERENCES agent_jobs(id), + success_count INTEGER NOT NULL DEFAULT 0, + failure_count INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + status TEXT NOT NULL DEFAULT 'active', + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_dynamic_tools_status ON dynamic_tools(status); +CREATE INDEX IF NOT EXISTS idx_dynamic_tools_name ON dynamic_tools(name); + +-- ==================== LLM Calls ==================== + +CREATE TABLE IF NOT EXISTS llm_calls ( + id TEXT PRIMARY KEY, + job_id TEXT REFERENCES agent_jobs(id) ON DELETE CASCADE, + conversation_id TEXT REFERENCES conversations(id), + provider TEXT NOT NULL, + model TEXT NOT NULL, + input_tokens INTEGER NOT NULL, + output_tokens INTEGER NOT NULL, + cost TEXT NOT NULL, + purpose TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_llm_calls_job ON llm_calls(job_id); +CREATE INDEX IF NOT EXISTS idx_llm_calls_conversation ON llm_calls(conversation_id); +CREATE INDEX IF NOT EXISTS idx_llm_calls_provider ON llm_calls(provider); + +-- ==================== Estimation ==================== + +CREATE TABLE IF NOT EXISTS estimation_snapshots ( + id TEXT PRIMARY KEY, + job_id TEXT NOT NULL REFERENCES agent_jobs(id) ON DELETE CASCADE, + category TEXT NOT NULL, + tool_names TEXT NOT NULL DEFAULT '[]', + estimated_cost TEXT NOT NULL, + actual_cost TEXT, + estimated_time_secs INTEGER NOT NULL, + actual_time_secs INTEGER, + estimated_value TEXT NOT NULL, + actual_value TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_estimation_category ON estimation_snapshots(category); +CREATE INDEX IF NOT EXISTS idx_estimation_job ON estimation_snapshots(job_id); + +-- ==================== Self Repair ==================== + +CREATE TABLE IF NOT EXISTS repair_attempts ( + id TEXT PRIMARY KEY, + target_type TEXT NOT NULL, + target_id TEXT NOT NULL, + diagnosis TEXT NOT NULL, + action_taken TEXT NOT NULL, + success INTEGER NOT NULL, + error_message TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_repair_attempts_target ON repair_attempts(target_type, target_id); +CREATE INDEX IF NOT EXISTS idx_repair_attempts_created ON repair_attempts(created_at); + +-- ==================== Workspace: Memory Documents ==================== + +CREATE TABLE IF NOT EXISTS memory_documents ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + agent_id TEXT, + path TEXT NOT NULL, + content TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + metadata TEXT NOT NULL DEFAULT '{}', + UNIQUE (user_id, agent_id, path) +); + +CREATE INDEX IF NOT EXISTS idx_memory_documents_user ON memory_documents(user_id); +CREATE INDEX IF NOT EXISTS idx_memory_documents_path ON memory_documents(user_id, path); +CREATE INDEX IF NOT EXISTS idx_memory_documents_updated ON memory_documents(updated_at DESC); + +-- Trigger to auto-update updated_at on memory_documents +CREATE TRIGGER IF NOT EXISTS update_memory_documents_updated_at + AFTER UPDATE ON memory_documents + FOR EACH ROW + WHEN NEW.updated_at = OLD.updated_at + BEGIN + UPDATE memory_documents SET updated_at = datetime('now') WHERE id = NEW.id; + END; + +-- ==================== Workspace: Memory Chunks ==================== + +CREATE TABLE IF NOT EXISTS memory_chunks ( + _rowid INTEGER PRIMARY KEY AUTOINCREMENT, + id TEXT NOT NULL UNIQUE, + document_id TEXT NOT NULL REFERENCES memory_documents(id) ON DELETE CASCADE, + chunk_index INTEGER NOT NULL, + content TEXT NOT NULL, + embedding F32_BLOB(1536), + created_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE (document_id, chunk_index) +); + +CREATE INDEX IF NOT EXISTS idx_memory_chunks_document ON memory_chunks(document_id); + +-- Vector index for semantic search (libSQL native) +CREATE INDEX IF NOT EXISTS idx_memory_chunks_embedding + ON memory_chunks (libsql_vector_idx(embedding)); + +-- FTS5 virtual table for full-text search +CREATE VIRTUAL TABLE IF NOT EXISTS memory_chunks_fts USING fts5( + content, + content='memory_chunks', + content_rowid='_rowid' +); + +-- Triggers to keep FTS5 in sync with memory_chunks +CREATE TRIGGER IF NOT EXISTS memory_chunks_fts_insert AFTER INSERT ON memory_chunks BEGIN + INSERT INTO memory_chunks_fts(rowid, content) VALUES (new._rowid, new.content); +END; + +CREATE TRIGGER IF NOT EXISTS memory_chunks_fts_delete AFTER DELETE ON memory_chunks BEGIN + INSERT INTO memory_chunks_fts(memory_chunks_fts, rowid, content) + VALUES ('delete', old._rowid, old.content); +END; + +CREATE TRIGGER IF NOT EXISTS memory_chunks_fts_update AFTER UPDATE ON memory_chunks BEGIN + INSERT INTO memory_chunks_fts(memory_chunks_fts, rowid, content) + VALUES ('delete', old._rowid, old.content); + INSERT INTO memory_chunks_fts(rowid, content) VALUES (new._rowid, new.content); +END; + +-- ==================== Workspace: Heartbeat State ==================== + +CREATE TABLE IF NOT EXISTS heartbeat_state ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + agent_id TEXT, + last_run TEXT, + next_run TEXT, + interval_seconds INTEGER NOT NULL DEFAULT 1800, + enabled INTEGER NOT NULL DEFAULT 1, + consecutive_failures INTEGER NOT NULL DEFAULT 0, + last_checks TEXT NOT NULL DEFAULT '{}', + UNIQUE (user_id, agent_id) +); + +CREATE INDEX IF NOT EXISTS idx_heartbeat_user ON heartbeat_state(user_id); + +-- ==================== Secrets ==================== + +CREATE TABLE IF NOT EXISTS secrets ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + name TEXT NOT NULL, + encrypted_value BLOB NOT NULL, + key_salt BLOB NOT NULL, + provider TEXT, + expires_at TEXT, + last_used_at TEXT, + usage_count INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE (user_id, name) +); + +CREATE INDEX IF NOT EXISTS idx_secrets_user ON secrets(user_id); + +-- ==================== WASM Tools ==================== + +CREATE TABLE IF NOT EXISTS wasm_tools ( + id TEXT PRIMARY KEY, + user_id TEXT NOT NULL, + name TEXT NOT NULL, + version TEXT NOT NULL DEFAULT '1.0.0', + description TEXT NOT NULL, + wasm_binary BLOB NOT NULL, + binary_hash BLOB NOT NULL, + parameters_schema TEXT NOT NULL, + source_url TEXT, + trust_level TEXT NOT NULL DEFAULT 'user', + status TEXT NOT NULL DEFAULT 'active', + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE (user_id, name, version) +); + +CREATE INDEX IF NOT EXISTS idx_wasm_tools_user ON wasm_tools(user_id); +CREATE INDEX IF NOT EXISTS idx_wasm_tools_name ON wasm_tools(user_id, name); +CREATE INDEX IF NOT EXISTS idx_wasm_tools_status ON wasm_tools(status); + +-- ==================== Tool Capabilities ==================== + +CREATE TABLE IF NOT EXISTS tool_capabilities ( + id TEXT PRIMARY KEY, + wasm_tool_id TEXT NOT NULL REFERENCES wasm_tools(id) ON DELETE CASCADE, + http_allowlist TEXT NOT NULL DEFAULT '[]', + allowed_secrets TEXT NOT NULL DEFAULT '[]', + tool_aliases TEXT NOT NULL DEFAULT '{}', + requests_per_minute INTEGER NOT NULL DEFAULT 60, + requests_per_hour INTEGER NOT NULL DEFAULT 1000, + max_request_body_bytes INTEGER NOT NULL DEFAULT 1048576, + max_response_body_bytes INTEGER NOT NULL DEFAULT 10485760, + workspace_read_prefixes TEXT NOT NULL DEFAULT '[]', + http_timeout_secs INTEGER NOT NULL DEFAULT 30, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE (wasm_tool_id) +); + +-- ==================== Leak Detection Patterns ==================== + +CREATE TABLE IF NOT EXISTS leak_detection_patterns ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE, + pattern TEXT NOT NULL, + severity TEXT NOT NULL DEFAULT 'high', + action TEXT NOT NULL DEFAULT 'block', + enabled INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- ==================== Rate Limit State ==================== + +CREATE TABLE IF NOT EXISTS tool_rate_limit_state ( + id TEXT PRIMARY KEY, + wasm_tool_id TEXT NOT NULL REFERENCES wasm_tools(id) ON DELETE CASCADE, + user_id TEXT NOT NULL, + minute_window_start TEXT NOT NULL DEFAULT (datetime('now')), + minute_count INTEGER NOT NULL DEFAULT 0, + hour_window_start TEXT NOT NULL DEFAULT (datetime('now')), + hour_count INTEGER NOT NULL DEFAULT 0, + UNIQUE (wasm_tool_id, user_id) +); + +-- ==================== Secret Usage Audit Log ==================== + +CREATE TABLE IF NOT EXISTS secret_usage_log ( + id TEXT PRIMARY KEY, + secret_id TEXT NOT NULL REFERENCES secrets(id) ON DELETE CASCADE, + wasm_tool_id TEXT REFERENCES wasm_tools(id) ON DELETE SET NULL, + user_id TEXT NOT NULL, + target_host TEXT NOT NULL, + target_path TEXT, + success INTEGER NOT NULL, + error_message TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_secret_usage_user ON secret_usage_log(user_id); + +-- ==================== Leak Detection Events ==================== + +CREATE TABLE IF NOT EXISTS leak_detection_events ( + id TEXT PRIMARY KEY, + pattern_id TEXT REFERENCES leak_detection_patterns(id) ON DELETE SET NULL, + wasm_tool_id TEXT REFERENCES wasm_tools(id) ON DELETE SET NULL, + user_id TEXT NOT NULL, + source TEXT NOT NULL, + action_taken TEXT NOT NULL, + context_preview TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +-- ==================== Tool Failures ==================== + +CREATE TABLE IF NOT EXISTS tool_failures ( + id TEXT PRIMARY KEY, + tool_name TEXT NOT NULL UNIQUE, + error_message TEXT, + error_count INTEGER DEFAULT 1, + first_failure TEXT DEFAULT (datetime('now')), + last_failure TEXT DEFAULT (datetime('now')), + last_build_result TEXT, + repaired_at TEXT, + repair_attempts INTEGER DEFAULT 0 +); + +CREATE INDEX IF NOT EXISTS idx_tool_failures_name ON tool_failures(tool_name); + +-- ==================== Job Events ==================== + +CREATE TABLE IF NOT EXISTS job_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + job_id TEXT NOT NULL REFERENCES agent_jobs(id), + event_type TEXT NOT NULL, + data TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_job_events_job ON job_events(job_id, id); + +-- ==================== Routines ==================== + +CREATE TABLE IF NOT EXISTS routines ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + user_id TEXT NOT NULL, + enabled INTEGER NOT NULL DEFAULT 1, + trigger_type TEXT NOT NULL, + trigger_config TEXT NOT NULL, + action_type TEXT NOT NULL, + action_config TEXT NOT NULL, + cooldown_secs INTEGER NOT NULL DEFAULT 300, + max_concurrent INTEGER NOT NULL DEFAULT 1, + dedup_window_secs INTEGER, + notify_channel TEXT, + notify_user TEXT NOT NULL DEFAULT 'default', + notify_on_success INTEGER NOT NULL DEFAULT 0, + notify_on_failure INTEGER NOT NULL DEFAULT 1, + notify_on_attention INTEGER NOT NULL DEFAULT 1, + state TEXT NOT NULL DEFAULT '{}', + last_run_at TEXT, + next_fire_at TEXT, + run_count INTEGER NOT NULL DEFAULT 0, + consecutive_failures INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE (user_id, name) +); + +CREATE INDEX IF NOT EXISTS idx_routines_user ON routines(user_id); + +-- ==================== Routine Runs ==================== + +CREATE TABLE IF NOT EXISTS routine_runs ( + id TEXT PRIMARY KEY, + routine_id TEXT NOT NULL REFERENCES routines(id) ON DELETE CASCADE, + trigger_type TEXT NOT NULL, + trigger_detail TEXT, + started_at TEXT NOT NULL DEFAULT (datetime('now')), + completed_at TEXT, + status TEXT NOT NULL DEFAULT 'running', + result_summary TEXT, + tokens_used INTEGER, + job_id TEXT REFERENCES agent_jobs(id), + created_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_routine_runs_routine ON routine_runs(routine_id); + +-- ==================== Settings ==================== + +CREATE TABLE IF NOT EXISTS settings ( + user_id TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (user_id, key) +); + +CREATE INDEX IF NOT EXISTS idx_settings_user ON settings(user_id); + +-- ==================== Missing indexes (parity with PostgreSQL) ==================== + +-- agent_jobs +CREATE INDEX IF NOT EXISTS idx_agent_jobs_stuck ON agent_jobs(stuck_since); + +-- secrets +CREATE INDEX IF NOT EXISTS idx_secrets_provider ON secrets(provider); +CREATE INDEX IF NOT EXISTS idx_secrets_expires ON secrets(expires_at); + +-- wasm_tools +CREATE INDEX IF NOT EXISTS idx_wasm_tools_trust ON wasm_tools(trust_level); + +-- tool_capabilities +CREATE INDEX IF NOT EXISTS idx_tool_capabilities_tool ON tool_capabilities(wasm_tool_id); + +-- leak_detection_patterns +CREATE INDEX IF NOT EXISTS idx_leak_patterns_enabled ON leak_detection_patterns(enabled); + +-- tool_rate_limit_state +CREATE INDEX IF NOT EXISTS idx_rate_limit_tool ON tool_rate_limit_state(wasm_tool_id); + +-- secret_usage_log +CREATE INDEX IF NOT EXISTS idx_secret_usage_secret ON secret_usage_log(secret_id); +CREATE INDEX IF NOT EXISTS idx_secret_usage_tool ON secret_usage_log(wasm_tool_id); +CREATE INDEX IF NOT EXISTS idx_secret_usage_created ON secret_usage_log(created_at DESC); + +-- leak_detection_events +CREATE INDEX IF NOT EXISTS idx_leak_events_pattern ON leak_detection_events(pattern_id); +CREATE INDEX IF NOT EXISTS idx_leak_events_tool ON leak_detection_events(wasm_tool_id); +CREATE INDEX IF NOT EXISTS idx_leak_events_user ON leak_detection_events(user_id); +CREATE INDEX IF NOT EXISTS idx_leak_events_created ON leak_detection_events(created_at DESC); + +-- tool_failures +CREATE INDEX IF NOT EXISTS idx_tool_failures_count ON tool_failures(error_count DESC); +CREATE INDEX IF NOT EXISTS idx_tool_failures_unrepaired ON tool_failures(tool_name); + +-- routines +CREATE INDEX IF NOT EXISTS idx_routines_next_fire ON routines(next_fire_at); +CREATE INDEX IF NOT EXISTS idx_routines_event_triggers ON routines(user_id); + +-- routine_runs +CREATE INDEX IF NOT EXISTS idx_routine_runs_status ON routine_runs(status); + +-- heartbeat_state +CREATE INDEX IF NOT EXISTS idx_heartbeat_next_run ON heartbeat_state(next_run); + +-- ==================== Seed data ==================== + +-- Pre-populate leak detection patterns (matches PostgreSQL V2 migration). +INSERT OR IGNORE INTO leak_detection_patterns (id, name, pattern, severity, action, enabled, created_at) VALUES + ('550e8400-e29b-41d4-a716-446655440001', 'openai_api_key', 'sk-(?:proj-)?[a-zA-Z0-9]{20,}(?:T3BlbkFJ[a-zA-Z0-9_-]*)?', 'critical', 'block', 1, datetime('now')), + ('550e8400-e29b-41d4-a716-446655440002', 'anthropic_api_key', 'sk-ant-api[a-zA-Z0-9_-]{90,}', 'critical', 'block', 1, datetime('now')), + ('550e8400-e29b-41d4-a716-446655440003', 'aws_access_key', 'AKIA[0-9A-Z]{16}', 'critical', 'block', 1, datetime('now')), + ('550e8400-e29b-41d4-a716-446655440004', 'aws_secret_key', '(?`. + +#[cfg(feature = "postgres")] +pub mod postgres; + +#[cfg(feature = "libsql")] +pub mod libsql_backend; + +#[cfg(feature = "libsql")] +pub mod libsql_migrations; + +use std::collections::HashMap; +use std::sync::Arc; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use rust_decimal::Decimal; +use uuid::Uuid; + +use crate::agent::BrokenTool; +use crate::agent::routine::{Routine, RoutineRun, RunStatus}; +use crate::context::{ActionRecord, JobContext, JobState}; +use crate::error::DatabaseError; +use crate::error::WorkspaceError; +use crate::history::{ + ConversationMessage, ConversationSummary, JobEventRecord, LlmCallRecord, SandboxJobRecord, + SandboxJobSummary, SettingRow, +}; +use crate::workspace::{MemoryChunk, MemoryDocument, WorkspaceEntry}; +use crate::workspace::{SearchConfig, SearchResult}; + +/// Create a database backend from configuration, run migrations, and return it. +/// +/// This is the shared helper for CLI commands and other call sites that need +/// a simple `Arc` without retaining backend-specific handles +/// (e.g., `pg_pool` or `libsql_conn` for the secrets store). The main agent +/// startup in `main.rs` uses its own initialization block because it also +/// captures those backend-specific handles. +pub async fn connect_from_config( + config: &crate::config::DatabaseConfig, +) -> Result, DatabaseError> { + match config.backend { + #[cfg(feature = "libsql")] + crate::config::DatabaseBackend::LibSql => { + use secrecy::ExposeSecret as _; + + let default_path = crate::config::default_libsql_path(); + let db_path = config.libsql_path.as_deref().unwrap_or(&default_path); + + let backend = if let Some(ref url) = config.libsql_url { + let token = config.libsql_auth_token.as_ref().ok_or_else(|| { + DatabaseError::Pool( + "LIBSQL_AUTH_TOKEN required when LIBSQL_URL is set".to_string(), + ) + })?; + libsql_backend::LibSqlBackend::new_remote_replica( + db_path, + url, + token.expose_secret(), + ) + .await + .map_err(|e| DatabaseError::Pool(e.to_string()))? + } else { + libsql_backend::LibSqlBackend::new_local(db_path) + .await + .map_err(|e| DatabaseError::Pool(e.to_string()))? + }; + backend.run_migrations().await?; + Ok(Arc::new(backend)) + } + #[cfg(feature = "postgres")] + _ => { + let pg = postgres::PgBackend::new(config) + .await + .map_err(|e| DatabaseError::Pool(e.to_string()))?; + pg.run_migrations().await?; + Ok(Arc::new(pg)) + } + #[cfg(not(feature = "postgres"))] + _ => Err(DatabaseError::Pool( + "No database backend available. Enable 'postgres' or 'libsql' feature.".to_string(), + )), + } +} + +/// Backend-agnostic database trait. +/// +/// Combines all persistence operations from Store, Repository, and related +/// stores into a single trait that can be implemented for different backends. +#[async_trait] +pub trait Database: Send + Sync { + /// Run schema migrations for this backend. + async fn run_migrations(&self) -> Result<(), DatabaseError>; + + // ==================== Conversations ==================== + + /// Create a new conversation. + async fn create_conversation( + &self, + channel: &str, + user_id: &str, + thread_id: Option<&str>, + ) -> Result; + + /// Update conversation last activity. + async fn touch_conversation(&self, id: Uuid) -> Result<(), DatabaseError>; + + /// Add a message to a conversation. + async fn add_conversation_message( + &self, + conversation_id: Uuid, + role: &str, + content: &str, + ) -> Result; + + /// Ensure a conversation row exists (upsert). + async fn ensure_conversation( + &self, + id: Uuid, + channel: &str, + user_id: &str, + thread_id: Option<&str>, + ) -> Result<(), DatabaseError>; + + /// List conversations with a title preview. + async fn list_conversations_with_preview( + &self, + user_id: &str, + channel: &str, + limit: i64, + ) -> Result, DatabaseError>; + + /// Get or create the singleton assistant conversation. + async fn get_or_create_assistant_conversation( + &self, + user_id: &str, + channel: &str, + ) -> Result; + + /// Create a conversation with specific metadata. + async fn create_conversation_with_metadata( + &self, + channel: &str, + user_id: &str, + metadata: &serde_json::Value, + ) -> Result; + + /// Load messages with cursor-based pagination. + async fn list_conversation_messages_paginated( + &self, + conversation_id: Uuid, + before: Option>, + limit: i64, + ) -> Result<(Vec, bool), DatabaseError>; + + /// Merge a single key into conversation metadata. + async fn update_conversation_metadata_field( + &self, + id: Uuid, + key: &str, + value: &serde_json::Value, + ) -> Result<(), DatabaseError>; + + /// Read conversation metadata. + async fn get_conversation_metadata( + &self, + id: Uuid, + ) -> Result, DatabaseError>; + + /// Load all messages for a conversation. + async fn list_conversation_messages( + &self, + conversation_id: Uuid, + ) -> Result, DatabaseError>; + + /// Check if a conversation belongs to a specific user. + async fn conversation_belongs_to_user( + &self, + conversation_id: Uuid, + user_id: &str, + ) -> Result; + + // ==================== Jobs ==================== + + /// Save a job context. + async fn save_job(&self, ctx: &JobContext) -> Result<(), DatabaseError>; + + /// Get a job by ID. + async fn get_job(&self, id: Uuid) -> Result, DatabaseError>; + + /// Update job status. + async fn update_job_status( + &self, + id: Uuid, + status: JobState, + failure_reason: Option<&str>, + ) -> Result<(), DatabaseError>; + + /// Mark job as stuck. + async fn mark_job_stuck(&self, id: Uuid) -> Result<(), DatabaseError>; + + /// Get stuck jobs. + async fn get_stuck_jobs(&self) -> Result, DatabaseError>; + + // ==================== Actions ==================== + + /// Save a job action. + async fn save_action(&self, job_id: Uuid, action: &ActionRecord) -> Result<(), DatabaseError>; + + /// Get actions for a job. + async fn get_job_actions(&self, job_id: Uuid) -> Result, DatabaseError>; + + // ==================== LLM Calls ==================== + + /// Record an LLM call. + async fn record_llm_call(&self, record: &LlmCallRecord<'_>) -> Result; + + // ==================== Estimation Snapshots ==================== + + /// Save an estimation snapshot. + async fn save_estimation_snapshot( + &self, + job_id: Uuid, + category: &str, + tool_names: &[String], + estimated_cost: Decimal, + estimated_time_secs: i32, + estimated_value: Decimal, + ) -> Result; + + /// Update estimation snapshot with actual values. + async fn update_estimation_actuals( + &self, + id: Uuid, + actual_cost: Decimal, + actual_time_secs: i32, + actual_value: Option, + ) -> Result<(), DatabaseError>; + + // ==================== Sandbox Jobs ==================== + + /// Insert a new sandbox job. + async fn save_sandbox_job(&self, job: &SandboxJobRecord) -> Result<(), DatabaseError>; + + /// Get a sandbox job by ID. + async fn get_sandbox_job(&self, id: Uuid) -> Result, DatabaseError>; + + /// List all sandbox jobs, most recent first. + async fn list_sandbox_jobs(&self) -> Result, DatabaseError>; + + /// Update sandbox job status. + async fn update_sandbox_job_status( + &self, + id: Uuid, + status: &str, + success: Option, + message: Option<&str>, + started_at: Option>, + completed_at: Option>, + ) -> Result<(), DatabaseError>; + + /// Mark stale sandbox jobs as interrupted. + async fn cleanup_stale_sandbox_jobs(&self) -> Result; + + /// Get sandbox job summary. + async fn sandbox_job_summary(&self) -> Result; + + /// List sandbox jobs for a specific user, most recent first. + async fn list_sandbox_jobs_for_user( + &self, + user_id: &str, + ) -> Result, DatabaseError>; + + /// Get sandbox job summary for a specific user. + async fn sandbox_job_summary_for_user( + &self, + user_id: &str, + ) -> Result; + + /// Check if a sandbox job belongs to a specific user. + async fn sandbox_job_belongs_to_user( + &self, + job_id: Uuid, + user_id: &str, + ) -> Result; + + /// Update sandbox job mode. + async fn update_sandbox_job_mode(&self, id: Uuid, mode: &str) -> Result<(), DatabaseError>; + + /// Get sandbox job mode. + async fn get_sandbox_job_mode(&self, id: Uuid) -> Result, DatabaseError>; + + // ==================== Job Events ==================== + + /// Persist a job event. + async fn save_job_event( + &self, + job_id: Uuid, + event_type: &str, + data: &serde_json::Value, + ) -> Result<(), DatabaseError>; + + /// Load all job events. + async fn list_job_events(&self, job_id: Uuid) -> Result, DatabaseError>; + + // ==================== Routines ==================== + + /// Create a new routine. + async fn create_routine(&self, routine: &Routine) -> Result<(), DatabaseError>; + + /// Get a routine by ID. + async fn get_routine(&self, id: Uuid) -> Result, DatabaseError>; + + /// Get a routine by user_id and name. + async fn get_routine_by_name( + &self, + user_id: &str, + name: &str, + ) -> Result, DatabaseError>; + + /// List routines for a user. + async fn list_routines(&self, user_id: &str) -> Result, DatabaseError>; + + /// List all enabled event routines. + async fn list_event_routines(&self) -> Result, DatabaseError>; + + /// List due cron routines. + async fn list_due_cron_routines(&self) -> Result, DatabaseError>; + + /// Update a routine. + async fn update_routine(&self, routine: &Routine) -> Result<(), DatabaseError>; + + /// Update runtime state after a routine fires. + async fn update_routine_runtime( + &self, + id: Uuid, + last_run_at: DateTime, + next_fire_at: Option>, + run_count: u64, + consecutive_failures: u32, + state: &serde_json::Value, + ) -> Result<(), DatabaseError>; + + /// Delete a routine. + async fn delete_routine(&self, id: Uuid) -> Result; + + // ==================== Routine Runs ==================== + + /// Record a routine run starting. + async fn create_routine_run(&self, run: &RoutineRun) -> Result<(), DatabaseError>; + + /// Complete a routine run. + async fn complete_routine_run( + &self, + id: Uuid, + status: RunStatus, + result_summary: Option<&str>, + tokens_used: Option, + ) -> Result<(), DatabaseError>; + + /// List recent runs for a routine. + async fn list_routine_runs( + &self, + routine_id: Uuid, + limit: i64, + ) -> Result, DatabaseError>; + + /// Count currently running runs for a routine. + async fn count_running_routine_runs(&self, routine_id: Uuid) -> Result; + + // ==================== Tool Failures ==================== + + /// Record a tool failure (upsert). + async fn record_tool_failure( + &self, + tool_name: &str, + error_message: &str, + ) -> Result<(), DatabaseError>; + + /// Get broken tools exceeding threshold. + async fn get_broken_tools(&self, threshold: i32) -> Result, DatabaseError>; + + /// Mark a tool as repaired. + async fn mark_tool_repaired(&self, tool_name: &str) -> Result<(), DatabaseError>; + + /// Increment repair attempts. + async fn increment_repair_attempts(&self, tool_name: &str) -> Result<(), DatabaseError>; + + // ==================== Settings ==================== + + /// Get a single setting. + async fn get_setting( + &self, + user_id: &str, + key: &str, + ) -> Result, DatabaseError>; + + /// Get a single setting with metadata. + async fn get_setting_full( + &self, + user_id: &str, + key: &str, + ) -> Result, DatabaseError>; + + /// Set a single setting (upsert). + async fn set_setting( + &self, + user_id: &str, + key: &str, + value: &serde_json::Value, + ) -> Result<(), DatabaseError>; + + /// Delete a single setting. + async fn delete_setting(&self, user_id: &str, key: &str) -> Result; + + /// List all settings for a user. + async fn list_settings(&self, user_id: &str) -> Result, DatabaseError>; + + /// Get all settings as a flat map. + async fn get_all_settings( + &self, + user_id: &str, + ) -> Result, DatabaseError>; + + /// Bulk-write settings atomically. + async fn set_all_settings( + &self, + user_id: &str, + settings: &HashMap, + ) -> Result<(), DatabaseError>; + + /// Check if settings exist for a user. + async fn has_settings(&self, user_id: &str) -> Result; + + // ==================== Workspace: Documents ==================== + + /// Get a document by path. + async fn get_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result; + + /// Get a document by ID. + async fn get_document_by_id(&self, id: Uuid) -> Result; + + /// Get or create a document by path. + async fn get_or_create_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result; + + /// Update a document's content. + async fn update_document(&self, id: Uuid, content: &str) -> Result<(), WorkspaceError>; + + /// Delete a document by path. + async fn delete_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result<(), WorkspaceError>; + + /// List files and directories in a directory path. + async fn list_directory( + &self, + user_id: &str, + agent_id: Option, + directory: &str, + ) -> Result, WorkspaceError>; + + /// List all file paths in the workspace. + async fn list_all_paths( + &self, + user_id: &str, + agent_id: Option, + ) -> Result, WorkspaceError>; + + /// List all documents for a user. + async fn list_documents( + &self, + user_id: &str, + agent_id: Option, + ) -> Result, WorkspaceError>; + + // ==================== Workspace: Chunks ==================== + + /// Delete all chunks for a document. + async fn delete_chunks(&self, document_id: Uuid) -> Result<(), WorkspaceError>; + + /// Insert a chunk. + async fn insert_chunk( + &self, + document_id: Uuid, + chunk_index: i32, + content: &str, + embedding: Option<&[f32]>, + ) -> Result; + + /// Update a chunk's embedding. + async fn update_chunk_embedding( + &self, + chunk_id: Uuid, + embedding: &[f32], + ) -> Result<(), WorkspaceError>; + + /// Get chunks without embeddings for backfilling. + async fn get_chunks_without_embeddings( + &self, + user_id: &str, + agent_id: Option, + limit: usize, + ) -> Result, WorkspaceError>; + + // ==================== Workspace: Search ==================== + + /// Perform hybrid search combining FTS and vector similarity. + async fn hybrid_search( + &self, + user_id: &str, + agent_id: Option, + query: &str, + embedding: Option<&[f32]>, + config: &SearchConfig, + ) -> Result, WorkspaceError>; +} diff --git a/src/db/postgres.rs b/src/db/postgres.rs new file mode 100644 index 00000000..9676144c --- /dev/null +++ b/src/db/postgres.rs @@ -0,0 +1,627 @@ +//! PostgreSQL backend for the Database trait. +//! +//! Delegates to the existing `Store` (history) and `Repository` (workspace) +//! implementations, avoiding SQL duplication. + +use std::collections::HashMap; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use deadpool_postgres::Pool; +use rust_decimal::Decimal; +use uuid::Uuid; + +use crate::agent::BrokenTool; +use crate::agent::routine::{Routine, RoutineRun, RunStatus}; +use crate::config::DatabaseConfig; +use crate::context::{ActionRecord, JobContext, JobState}; +use crate::db::Database; +use crate::error::{DatabaseError, WorkspaceError}; +use crate::history::{ + ConversationMessage, ConversationSummary, JobEventRecord, LlmCallRecord, SandboxJobRecord, + SandboxJobSummary, SettingRow, Store, +}; +use crate::workspace::{ + MemoryChunk, MemoryDocument, Repository, SearchConfig, SearchResult, WorkspaceEntry, +}; + +/// PostgreSQL database backend. +/// +/// Wraps the existing `Store` (for history/conversations/jobs/routines/settings) +/// and `Repository` (for workspace documents/chunks/search) to implement the +/// unified `Database` trait. +pub struct PgBackend { + store: Store, + repo: Repository, +} + +impl PgBackend { + /// Create a new PostgreSQL backend from configuration. + pub async fn new(config: &DatabaseConfig) -> Result { + let store = Store::new(config).await?; + let repo = Repository::new(store.pool()); + Ok(Self { store, repo }) + } + + /// Get a clone of the connection pool. + /// + /// Useful for sharing with components that still need raw pool access. + pub fn pool(&self) -> Pool { + self.store.pool() + } +} + +#[async_trait] +impl Database for PgBackend { + async fn run_migrations(&self) -> Result<(), DatabaseError> { + self.store.run_migrations().await + } + + // ==================== Conversations ==================== + + async fn create_conversation( + &self, + channel: &str, + user_id: &str, + thread_id: Option<&str>, + ) -> Result { + self.store + .create_conversation(channel, user_id, thread_id) + .await + } + + async fn touch_conversation(&self, id: Uuid) -> Result<(), DatabaseError> { + self.store.touch_conversation(id).await + } + + async fn add_conversation_message( + &self, + conversation_id: Uuid, + role: &str, + content: &str, + ) -> Result { + self.store + .add_conversation_message(conversation_id, role, content) + .await + } + + async fn ensure_conversation( + &self, + id: Uuid, + channel: &str, + user_id: &str, + thread_id: Option<&str>, + ) -> Result<(), DatabaseError> { + self.store + .ensure_conversation(id, channel, user_id, thread_id) + .await + } + + async fn list_conversations_with_preview( + &self, + user_id: &str, + channel: &str, + limit: i64, + ) -> Result, DatabaseError> { + self.store + .list_conversations_with_preview(user_id, channel, limit) + .await + } + + async fn get_or_create_assistant_conversation( + &self, + user_id: &str, + channel: &str, + ) -> Result { + self.store + .get_or_create_assistant_conversation(user_id, channel) + .await + } + + async fn create_conversation_with_metadata( + &self, + channel: &str, + user_id: &str, + metadata: &serde_json::Value, + ) -> Result { + self.store + .create_conversation_with_metadata(channel, user_id, metadata) + .await + } + + async fn list_conversation_messages_paginated( + &self, + conversation_id: Uuid, + before: Option>, + limit: i64, + ) -> Result<(Vec, bool), DatabaseError> { + self.store + .list_conversation_messages_paginated(conversation_id, before, limit) + .await + } + + async fn update_conversation_metadata_field( + &self, + id: Uuid, + key: &str, + value: &serde_json::Value, + ) -> Result<(), DatabaseError> { + self.store + .update_conversation_metadata_field(id, key, value) + .await + } + + async fn get_conversation_metadata( + &self, + id: Uuid, + ) -> Result, DatabaseError> { + self.store.get_conversation_metadata(id).await + } + + async fn list_conversation_messages( + &self, + conversation_id: Uuid, + ) -> Result, DatabaseError> { + self.store.list_conversation_messages(conversation_id).await + } + + async fn conversation_belongs_to_user( + &self, + conversation_id: Uuid, + user_id: &str, + ) -> Result { + self.store + .conversation_belongs_to_user(conversation_id, user_id) + .await + } + + // ==================== Jobs ==================== + + async fn save_job(&self, ctx: &JobContext) -> Result<(), DatabaseError> { + self.store.save_job(ctx).await + } + + async fn get_job(&self, id: Uuid) -> Result, DatabaseError> { + self.store.get_job(id).await + } + + async fn update_job_status( + &self, + id: Uuid, + status: JobState, + failure_reason: Option<&str>, + ) -> Result<(), DatabaseError> { + self.store + .update_job_status(id, status, failure_reason) + .await + } + + async fn mark_job_stuck(&self, id: Uuid) -> Result<(), DatabaseError> { + self.store.mark_job_stuck(id).await + } + + async fn get_stuck_jobs(&self) -> Result, DatabaseError> { + self.store.get_stuck_jobs().await + } + + // ==================== Actions ==================== + + async fn save_action(&self, job_id: Uuid, action: &ActionRecord) -> Result<(), DatabaseError> { + self.store.save_action(job_id, action).await + } + + async fn get_job_actions(&self, job_id: Uuid) -> Result, DatabaseError> { + self.store.get_job_actions(job_id).await + } + + // ==================== LLM Calls ==================== + + async fn record_llm_call(&self, record: &LlmCallRecord<'_>) -> Result { + self.store.record_llm_call(record).await + } + + // ==================== Estimation Snapshots ==================== + + async fn save_estimation_snapshot( + &self, + job_id: Uuid, + category: &str, + tool_names: &[String], + estimated_cost: Decimal, + estimated_time_secs: i32, + estimated_value: Decimal, + ) -> Result { + self.store + .save_estimation_snapshot( + job_id, + category, + tool_names, + estimated_cost, + estimated_time_secs, + estimated_value, + ) + .await + } + + async fn update_estimation_actuals( + &self, + id: Uuid, + actual_cost: Decimal, + actual_time_secs: i32, + actual_value: Option, + ) -> Result<(), DatabaseError> { + self.store + .update_estimation_actuals(id, actual_cost, actual_time_secs, actual_value) + .await + } + + // ==================== Sandbox Jobs ==================== + + async fn save_sandbox_job(&self, job: &SandboxJobRecord) -> Result<(), DatabaseError> { + self.store.save_sandbox_job(job).await + } + + async fn get_sandbox_job(&self, id: Uuid) -> Result, DatabaseError> { + self.store.get_sandbox_job(id).await + } + + async fn list_sandbox_jobs(&self) -> Result, DatabaseError> { + self.store.list_sandbox_jobs().await + } + + async fn update_sandbox_job_status( + &self, + id: Uuid, + status: &str, + success: Option, + message: Option<&str>, + started_at: Option>, + completed_at: Option>, + ) -> Result<(), DatabaseError> { + self.store + .update_sandbox_job_status(id, status, success, message, started_at, completed_at) + .await + } + + async fn cleanup_stale_sandbox_jobs(&self) -> Result { + self.store.cleanup_stale_sandbox_jobs().await + } + + async fn sandbox_job_summary(&self) -> Result { + self.store.sandbox_job_summary().await + } + + async fn list_sandbox_jobs_for_user( + &self, + user_id: &str, + ) -> Result, DatabaseError> { + self.store.list_sandbox_jobs_for_user(user_id).await + } + + async fn sandbox_job_summary_for_user( + &self, + user_id: &str, + ) -> Result { + self.store.sandbox_job_summary_for_user(user_id).await + } + + async fn sandbox_job_belongs_to_user( + &self, + job_id: Uuid, + user_id: &str, + ) -> Result { + self.store + .sandbox_job_belongs_to_user(job_id, user_id) + .await + } + + async fn update_sandbox_job_mode(&self, id: Uuid, mode: &str) -> Result<(), DatabaseError> { + self.store.update_sandbox_job_mode(id, mode).await + } + + async fn get_sandbox_job_mode(&self, id: Uuid) -> Result, DatabaseError> { + self.store.get_sandbox_job_mode(id).await + } + + // ==================== Job Events ==================== + + async fn save_job_event( + &self, + job_id: Uuid, + event_type: &str, + data: &serde_json::Value, + ) -> Result<(), DatabaseError> { + self.store.save_job_event(job_id, event_type, data).await + } + + async fn list_job_events(&self, job_id: Uuid) -> Result, DatabaseError> { + self.store.list_job_events(job_id).await + } + + // ==================== Routines ==================== + + async fn create_routine(&self, routine: &Routine) -> Result<(), DatabaseError> { + self.store.create_routine(routine).await + } + + async fn get_routine(&self, id: Uuid) -> Result, DatabaseError> { + self.store.get_routine(id).await + } + + async fn get_routine_by_name( + &self, + user_id: &str, + name: &str, + ) -> Result, DatabaseError> { + self.store.get_routine_by_name(user_id, name).await + } + + async fn list_routines(&self, user_id: &str) -> Result, DatabaseError> { + self.store.list_routines(user_id).await + } + + async fn list_event_routines(&self) -> Result, DatabaseError> { + self.store.list_event_routines().await + } + + async fn list_due_cron_routines(&self) -> Result, DatabaseError> { + self.store.list_due_cron_routines().await + } + + async fn update_routine(&self, routine: &Routine) -> Result<(), DatabaseError> { + self.store.update_routine(routine).await + } + + async fn update_routine_runtime( + &self, + id: Uuid, + last_run_at: DateTime, + next_fire_at: Option>, + run_count: u64, + consecutive_failures: u32, + state: &serde_json::Value, + ) -> Result<(), DatabaseError> { + self.store + .update_routine_runtime( + id, + last_run_at, + next_fire_at, + run_count, + consecutive_failures, + state, + ) + .await + } + + async fn delete_routine(&self, id: Uuid) -> Result { + self.store.delete_routine(id).await + } + + // ==================== Routine Runs ==================== + + async fn create_routine_run(&self, run: &RoutineRun) -> Result<(), DatabaseError> { + self.store.create_routine_run(run).await + } + + async fn complete_routine_run( + &self, + id: Uuid, + status: RunStatus, + result_summary: Option<&str>, + tokens_used: Option, + ) -> Result<(), DatabaseError> { + self.store + .complete_routine_run(id, status, result_summary, tokens_used) + .await + } + + async fn list_routine_runs( + &self, + routine_id: Uuid, + limit: i64, + ) -> Result, DatabaseError> { + self.store.list_routine_runs(routine_id, limit).await + } + + async fn count_running_routine_runs(&self, routine_id: Uuid) -> Result { + self.store.count_running_routine_runs(routine_id).await + } + + // ==================== Tool Failures ==================== + + async fn record_tool_failure( + &self, + tool_name: &str, + error_message: &str, + ) -> Result<(), DatabaseError> { + self.store + .record_tool_failure(tool_name, error_message) + .await + } + + async fn get_broken_tools(&self, threshold: i32) -> Result, DatabaseError> { + self.store.get_broken_tools(threshold).await + } + + async fn mark_tool_repaired(&self, tool_name: &str) -> Result<(), DatabaseError> { + self.store.mark_tool_repaired(tool_name).await + } + + async fn increment_repair_attempts(&self, tool_name: &str) -> Result<(), DatabaseError> { + self.store.increment_repair_attempts(tool_name).await + } + + // ==================== Settings ==================== + + async fn get_setting( + &self, + user_id: &str, + key: &str, + ) -> Result, DatabaseError> { + self.store.get_setting(user_id, key).await + } + + async fn get_setting_full( + &self, + user_id: &str, + key: &str, + ) -> Result, DatabaseError> { + self.store.get_setting_full(user_id, key).await + } + + async fn set_setting( + &self, + user_id: &str, + key: &str, + value: &serde_json::Value, + ) -> Result<(), DatabaseError> { + self.store.set_setting(user_id, key, value).await + } + + async fn delete_setting(&self, user_id: &str, key: &str) -> Result { + self.store.delete_setting(user_id, key).await + } + + async fn list_settings(&self, user_id: &str) -> Result, DatabaseError> { + self.store.list_settings(user_id).await + } + + async fn get_all_settings( + &self, + user_id: &str, + ) -> Result, DatabaseError> { + self.store.get_all_settings(user_id).await + } + + async fn set_all_settings( + &self, + user_id: &str, + settings: &HashMap, + ) -> Result<(), DatabaseError> { + self.store.set_all_settings(user_id, settings).await + } + + async fn has_settings(&self, user_id: &str) -> Result { + self.store.has_settings(user_id).await + } + + // ==================== Workspace: Documents ==================== + + async fn get_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result { + self.repo + .get_document_by_path(user_id, agent_id, path) + .await + } + + async fn get_document_by_id(&self, id: Uuid) -> Result { + self.repo.get_document_by_id(id).await + } + + async fn get_or_create_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result { + self.repo + .get_or_create_document_by_path(user_id, agent_id, path) + .await + } + + async fn update_document(&self, id: Uuid, content: &str) -> Result<(), WorkspaceError> { + self.repo.update_document(id, content).await + } + + async fn delete_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result<(), WorkspaceError> { + self.repo + .delete_document_by_path(user_id, agent_id, path) + .await + } + + async fn list_directory( + &self, + user_id: &str, + agent_id: Option, + directory: &str, + ) -> Result, WorkspaceError> { + self.repo.list_directory(user_id, agent_id, directory).await + } + + async fn list_all_paths( + &self, + user_id: &str, + agent_id: Option, + ) -> Result, WorkspaceError> { + self.repo.list_all_paths(user_id, agent_id).await + } + + async fn list_documents( + &self, + user_id: &str, + agent_id: Option, + ) -> Result, WorkspaceError> { + self.repo.list_documents(user_id, agent_id).await + } + + // ==================== Workspace: Chunks ==================== + + async fn delete_chunks(&self, document_id: Uuid) -> Result<(), WorkspaceError> { + self.repo.delete_chunks(document_id).await + } + + async fn insert_chunk( + &self, + document_id: Uuid, + chunk_index: i32, + content: &str, + embedding: Option<&[f32]>, + ) -> Result { + self.repo + .insert_chunk(document_id, chunk_index, content, embedding) + .await + } + + async fn update_chunk_embedding( + &self, + chunk_id: Uuid, + embedding: &[f32], + ) -> Result<(), WorkspaceError> { + self.repo.update_chunk_embedding(chunk_id, embedding).await + } + + async fn get_chunks_without_embeddings( + &self, + user_id: &str, + agent_id: Option, + limit: usize, + ) -> Result, WorkspaceError> { + self.repo + .get_chunks_without_embeddings(user_id, agent_id, limit) + .await + } + + // ==================== Workspace: Search ==================== + + async fn hybrid_search( + &self, + user_id: &str, + agent_id: Option, + query: &str, + embedding: Option<&[f32]>, + config: &SearchConfig, + ) -> Result, WorkspaceError> { + self.repo + .hybrid_search(user_id, agent_id, query, embedding, config) + .await + } +} diff --git a/src/error.rs b/src/error.rs index 189589d0..acb3598a 100644 --- a/src/error.rs +++ b/src/error.rs @@ -87,14 +87,21 @@ pub enum DatabaseError { #[error("Serialization error: {0}")] Serialization(String), + #[cfg(feature = "postgres")] #[error("PostgreSQL error: {0}")] Postgres(#[from] tokio_postgres::Error), + #[cfg(feature = "postgres")] #[error("Pool build error: {0}")] PoolBuild(#[from] deadpool_postgres::BuildError), + #[cfg(feature = "postgres")] #[error("Pool runtime error: {0}")] PoolRuntime(#[from] deadpool_postgres::PoolError), + + #[cfg(feature = "libsql")] + #[error("LibSQL error: {0}")] + LibSql(#[from] libsql::Error), } /// Channel-related errors. diff --git a/src/extensions/manager.rs b/src/extensions/manager.rs index d8ecefc6..e6e7d264 100644 --- a/src/extensions/manager.rs +++ b/src/extensions/manager.rs @@ -57,7 +57,7 @@ pub struct ExtensionManager { _tunnel_url: Option, user_id: String, /// Optional database store for DB-backed MCP config. - store: Option>, + store: Option>, } impl ExtensionManager { @@ -71,7 +71,7 @@ impl ExtensionManager { wasm_channels_dir: PathBuf, tunnel_url: Option, user_id: String, - store: Option>, + store: Option>, ) -> Self { Self { registry: ExtensionRegistry::new(), @@ -351,7 +351,7 @@ impl ExtensionManager { ) -> Result { if let Some(ref store) = self.store { - crate::tools::mcp::config::load_mcp_servers_from_db(store, &self.user_id).await + crate::tools::mcp::config::load_mcp_servers_from_db(store.as_ref(), &self.user_id).await } else { crate::tools::mcp::config::load_mcp_servers().await } @@ -375,7 +375,8 @@ impl ExtensionManager { ) -> Result<(), crate::tools::mcp::config::ConfigError> { config.validate()?; if let Some(ref store) = self.store { - crate::tools::mcp::config::add_mcp_server_db(store, &self.user_id, config).await + crate::tools::mcp::config::add_mcp_server_db(store.as_ref(), &self.user_id, config) + .await } else { crate::tools::mcp::config::add_mcp_server(config).await } @@ -386,7 +387,8 @@ impl ExtensionManager { name: &str, ) -> Result<(), crate::tools::mcp::config::ConfigError> { if let Some(ref store) = self.store { - crate::tools::mcp::config::remove_mcp_server_db(store, &self.user_id, name).await + crate::tools::mcp::config::remove_mcp_server_db(store.as_ref(), &self.user_id, name) + .await } else { crate::tools::mcp::config::remove_mcp_server(name).await } diff --git a/src/history/mod.rs b/src/history/mod.rs index a9e5df35..4f448cfc 100644 --- a/src/history/mod.rs +++ b/src/history/mod.rs @@ -5,11 +5,15 @@ //! - Learning from past executions //! - Analytics and metrics +#[cfg(feature = "postgres")] mod analytics; mod store; +#[cfg(feature = "postgres")] pub use analytics::{JobStats, ToolStats}; +#[cfg(feature = "postgres")] +pub use store::Store; pub use store::{ ConversationMessage, ConversationSummary, JobEventRecord, LlmCallRecord, SandboxJobRecord, - SandboxJobSummary, Store, + SandboxJobSummary, SettingRow, }; diff --git a/src/history/store.rs b/src/history/store.rs index 0f20bc62..17e85598 100644 --- a/src/history/store.rs +++ b/src/history/store.rs @@ -1,13 +1,18 @@ //! PostgreSQL store for persisting agent data. use chrono::{DateTime, Utc}; +#[cfg(feature = "postgres")] use deadpool_postgres::{Config, Pool, Runtime}; use rust_decimal::Decimal; +#[cfg(feature = "postgres")] use tokio_postgres::NoTls; use uuid::Uuid; +#[cfg(feature = "postgres")] use crate::config::DatabaseConfig; +#[cfg(feature = "postgres")] use crate::context::{ActionRecord, JobContext, JobState}; +#[cfg(feature = "postgres")] use crate::error::DatabaseError; /// Record for an LLM call to be persisted. @@ -24,10 +29,12 @@ pub struct LlmCallRecord<'a> { } /// Database store for the agent. +#[cfg(feature = "postgres")] pub struct Store { pool: Pool, } +#[cfg(feature = "postgres")] impl Store { /// Create a new store and connect to the database. pub async fn new(config: &DatabaseConfig) -> Result { @@ -144,7 +151,12 @@ impl Store { actual_cost, repair_attempts, created_at, started_at, completed_at ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) ON CONFLICT (id) DO UPDATE SET + title = EXCLUDED.title, + description = EXCLUDED.description, + category = EXCLUDED.category, status = EXCLUDED.status, + estimated_cost = EXCLUDED.estimated_cost, + estimated_time_secs = EXCLUDED.estimated_time_secs, actual_cost = EXCLUDED.actual_cost, repair_attempts = EXCLUDED.repair_attempts, started_at = EXCLUDED.started_at, @@ -466,6 +478,7 @@ pub struct SandboxJobSummary { pub interrupted: usize, } +#[cfg(feature = "postgres")] impl Store { /// Insert a new sandbox job into `agent_jobs`. pub async fn save_sandbox_job(&self, job: &SandboxJobRecord) -> Result<(), DatabaseError> { @@ -742,6 +755,7 @@ pub struct JobEventRecord { pub created_at: DateTime, } +#[cfg(feature = "postgres")] impl Store { /// Persist a job event (fire-and-forget from orchestrator handler). pub async fn save_job_event( @@ -814,10 +828,12 @@ impl Store { // ==================== Routines ==================== +#[cfg(feature = "postgres")] use crate::agent::routine::{ NotifyConfig, Routine, RoutineAction, RoutineGuardrails, RoutineRun, RunStatus, Trigger, }; +#[cfg(feature = "postgres")] impl Store { /// Create a new routine. pub async fn create_routine(&self, routine: &Routine) -> Result<(), DatabaseError> { @@ -1118,6 +1134,7 @@ impl Store { } } +#[cfg(feature = "postgres")] fn row_to_routine(row: &tokio_postgres::Row) -> Result { let trigger_type: String = row.get("trigger_type"); let trigger_config: serde_json::Value = row.get("trigger_config"); @@ -1162,6 +1179,7 @@ fn row_to_routine(row: &tokio_postgres::Row) -> Result { }) } +#[cfg(feature = "postgres")] fn row_to_routine_run(row: &tokio_postgres::Row) -> Result { let status_str: String = row.get("status"); let status: RunStatus = status_str @@ -1207,6 +1225,7 @@ pub struct ConversationMessage { pub created_at: DateTime, } +#[cfg(feature = "postgres")] impl Store { /// Ensure a conversation row exists for a given UUID. /// @@ -1477,6 +1496,7 @@ impl Store { } } +#[cfg(feature = "postgres")] fn parse_job_state(s: &str) -> JobState { match s { "pending" => JobState::Pending, @@ -1493,8 +1513,10 @@ fn parse_job_state(s: &str) -> JobState { // ==================== Tool Failures ==================== +#[cfg(feature = "postgres")] use crate::agent::BrokenTool; +#[cfg(feature = "postgres")] impl Store { /// Record a tool failure (upsert: increment count if exists). pub async fn record_tool_failure( @@ -1588,6 +1610,7 @@ pub struct SettingRow { pub updated_at: DateTime, } +#[cfg(feature = "postgres")] impl Store { /// Get a single setting by key. pub async fn get_setting( diff --git a/src/lib.rs b/src/lib.rs index 7185d3f3..ed72e3df 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -44,6 +44,7 @@ pub mod channels; pub mod cli; pub mod config; pub mod context; +pub mod db; pub mod error; pub mod estimation; pub mod evaluation; diff --git a/src/llm/session.rs b/src/llm/session.rs index 1350a5ac..c4f0ebc2 100644 --- a/src/llm/session.rs +++ b/src/llm/session.rs @@ -62,7 +62,7 @@ pub struct SessionManager { /// Prevents thundering herd during concurrent 401s. renewal_lock: Mutex<()>, /// Optional database store for persisting session to the settings table. - store: RwLock>>, + store: RwLock>>, /// User ID for DB settings (default: "default"). user_id: RwLock, } @@ -125,7 +125,7 @@ impl SessionManager { /// When a store is attached, session tokens are saved to the `settings` /// table (key: `nearai.session_token`) in addition to the disk file. /// On load, DB is preferred over disk. - pub async fn attach_store(&self, store: Arc, user_id: &str) { + pub async fn attach_store(&self, store: Arc, user_id: &str) { *self.store.write().await = Some(store); *self.user_id.write().await = user_id.to_string(); diff --git a/src/main.rs b/src/main.rs index cc619eb8..97355aaa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -17,13 +17,11 @@ use ironclaw::{ web::log_layer::{LogBroadcaster, WebLogLayer}, }, cli::{ - Cli, Command, run_mcp_command, run_memory_command, run_pairing_command, run_status_command, - run_tool_command, + Cli, Command, run_mcp_command, run_pairing_command, run_status_command, run_tool_command, }, config::Config, context::ContextManager, extensions::ExtensionManager, - history::Store, llm::{SessionConfig, create_llm_provider, create_session_manager}, orchestrator::{ ContainerJobConfig, ContainerJobManager, OrchestratorApi, TokenStore, @@ -31,8 +29,7 @@ use ironclaw::{ }, pairing::PairingStore, safety::SafetyLayer, - secrets::{PostgresSecretsStore, SecretsCrypto, SecretsStore}, - setup::{SetupConfig, SetupWizard}, + secrets::SecretsStore, tools::{ ToolRegistry, mcp::{McpClient, McpSessionManager, config::load_mcp_servers_from_db, is_authenticated}, @@ -41,6 +38,14 @@ use ironclaw::{ workspace::{EmbeddingProvider, NearAiEmbeddings, OpenAiEmbeddings, Workspace}, }; +#[cfg(feature = "libsql")] +use ironclaw::secrets::LibSqlSecretsStore; +#[cfg(feature = "postgres")] +use ironclaw::secrets::PostgresSecretsStore; +use ironclaw::secrets::SecretsCrypto; +#[cfg(any(feature = "postgres", feature = "libsql"))] +use ironclaw::setup::{SetupConfig, SetupWizard}; + #[tokio::main] async fn main() -> anyhow::Result<()> { let cli = Cli::parse(); @@ -89,8 +94,6 @@ async fn main() -> anyhow::Result<()> { let config = Config::from_env() .await .map_err(|e| anyhow::anyhow!("{}", e))?; - let store = ironclaw::history::Store::new(&config.database).await?; - store.run_migrations().await?; // Set up embeddings if available let session = ironclaw::llm::create_session_manager(ironclaw::llm::SessionConfig { @@ -130,7 +133,14 @@ async fn main() -> anyhow::Result<()> { None }; - return run_memory_command(mem_cmd.clone(), store.pool(), embeddings).await; + // Create a Database-trait-backed workspace for the memory command + let db: Arc = + ironclaw::db::connect_from_config(&config.database) + .await + .map_err(|e| anyhow::anyhow!("{}", e))?; + + return ironclaw::cli::run_memory_command_with_db(mem_cmd.clone(), db, embeddings) + .await; } Some(Command::Pairing(pairing_cmd)) => { tracing_subscriber::fmt() @@ -210,15 +220,13 @@ async fn main() -> anyhow::Result<()> { model ); - // Load allowed tools from config (env var or defaults). - let claude_config = ironclaw::config::ClaudeCodeConfig::from_env(); let config = ironclaw::worker::claude_bridge::ClaudeBridgeConfig { job_id: *job_id, orchestrator_url: orchestrator_url.clone(), max_turns: *max_turns, model: model.clone(), timeout: std::time::Duration::from_secs(1800), - allowed_tools: claude_config.allowed_tools, + allowed_tools: Vec::new(), }; let runtime = ironclaw::worker::ClaudeBridgeRuntime::new(config) @@ -238,12 +246,20 @@ async fn main() -> anyhow::Result<()> { // Load .env before running onboarding wizard let _ = dotenvy::dotenv(); - let config = SetupConfig { - skip_auth: *skip_auth, - channels_only: *channels_only, - }; - let mut wizard = SetupWizard::with_config(config); - wizard.run().await?; + #[cfg(any(feature = "postgres", feature = "libsql"))] + { + let config = SetupConfig { + skip_auth: *skip_auth, + channels_only: *channels_only, + }; + let mut wizard = SetupWizard::with_config(config); + wizard.run().await?; + } + #[cfg(not(any(feature = "postgres", feature = "libsql")))] + { + let _ = (skip_auth, channels_only); + eprintln!("Onboarding wizard requires the 'postgres' or 'libsql' feature."); + } return Ok(()); } None | Some(Command::Run) => { @@ -255,6 +271,7 @@ async fn main() -> anyhow::Result<()> { let _ = dotenvy::dotenv(); // Enhanced first-run detection + #[cfg(any(feature = "postgres", feature = "libsql"))] if !cli.no_onboard && let Some(reason) = check_onboard_needed().await { @@ -326,23 +343,86 @@ async fn main() -> anyhow::Result<()> { tracing::info!("Loaded configuration for agent: {}", config.agent.name); tracing::info!("LLM backend: {}", config.llm.backend); - // Initialize database store (optional for testing) - let store = if cli.no_db { + // Initialize database backend. + // + // Creates an `Arc` that all consumers share. + // Backend is selected by the `DATABASE_BACKEND` env var / config. + // + // NOTE: For simpler call sites (CLI commands, Memory handler) use the shared + // helper `ironclaw::db::connect_from_config()`. This block is kept inline + // because it also captures backend-specific handles (`pg_pool`, `libsql_db`) + // needed by the secrets store. + #[cfg(feature = "postgres")] + let mut pg_pool: Option = None; + #[cfg(feature = "libsql")] + let mut libsql_db: Option> = None; + + let db: Option> = if cli.no_db { tracing::warn!("Running without database connection"); None } else { - let store = Store::new(&config.database).await?; - store.run_migrations().await?; - tracing::info!("Database connected and migrations applied"); + match config.database.backend { + #[cfg(feature = "libsql")] + ironclaw::config::DatabaseBackend::LibSql => { + use ironclaw::db::Database as _; + use ironclaw::db::libsql_backend::LibSqlBackend; + use secrecy::ExposeSecret as _; + let default_path = ironclaw::config::default_libsql_path(); + let db_path = config + .database + .libsql_path + .as_deref() + .unwrap_or(&default_path); + + let backend = if let Some(ref url) = config.database.libsql_url { + let token = config.database.libsql_auth_token.as_ref().ok_or_else(|| { + anyhow::anyhow!("LIBSQL_AUTH_TOKEN is required when LIBSQL_URL is set") + })?; + LibSqlBackend::new_remote_replica(db_path, url, token.expose_secret()).await? + } else { + LibSqlBackend::new_local(db_path).await? + }; + backend.run_migrations().await?; + tracing::info!("libSQL database connected and migrations applied"); + + // Capture the Database handle for SecretsStore (connection-per-op) + libsql_db = Some(backend.shared_db()); + + Some(Arc::new(backend) as Arc) + } + #[cfg(feature = "postgres")] + _ => { + use ironclaw::db::Database as _; + let pg = ironclaw::db::postgres::PgBackend::new(&config.database) + .await + .map_err(|e| anyhow::anyhow!("{}", e))?; + pg.run_migrations() + .await + .map_err(|e| anyhow::anyhow!("{}", e))?; + tracing::info!("PostgreSQL database connected and migrations applied"); + + pg_pool = Some(pg.pool()); + Some(Arc::new(pg) as Arc) + } + #[cfg(not(feature = "postgres"))] + _ => { + anyhow::bail!( + "No database backend available. Enable 'postgres' or 'libsql' feature." + ); + } + } + }; + + // Post-init operations using the database + if let Some(ref db) = db { // One-time migration: move disk config files into the DB settings table. - if let Err(e) = ironclaw::bootstrap::migrate_disk_to_db(&store, "default").await { + if let Err(e) = ironclaw::bootstrap::migrate_disk_to_db(db.as_ref(), "default").await { tracing::warn!("Disk-to-DB settings migration failed: {}", e); } // Reload config from DB now that we have a connection. - // Priority: env var > DB setting > default. - match Config::from_db(&store, "default", &bootstrap).await { + match Config::from_db(db.as_ref(), "default", &bootstrap).await { Ok(db_config) => { config = db_config; tracing::info!("Configuration reloaded from database"); @@ -355,19 +435,14 @@ async fn main() -> anyhow::Result<()> { } } - let store = Arc::new(store); - - // Attach store to session manager so tokens save to DB too - session.attach_store(Arc::clone(&store), "default").await; + // Attach DB to session manager so tokens save to DB too + session.attach_store(Arc::clone(db), "default").await; // Mark any jobs left in "running" or "creating" state as "interrupted". - if let Err(e) = store.cleanup_stale_sandbox_jobs().await { + if let Err(e) = db.cleanup_stale_sandbox_jobs().await { tracing::warn!("Failed to cleanup stale sandbox jobs: {}", e); } - - Some(store) - }; - + } // Initialize LLM provider (clone session so we can reuse it for embeddings) let llm = create_llm_provider(&config.llm, session.clone())?; tracing::info!("LLM provider initialized: {}", llm.model_name()); @@ -421,8 +496,8 @@ async fn main() -> anyhow::Result<()> { }; // Register memory tools if database is available - if let Some(ref store) = store { - let mut workspace = Workspace::new("default", store.pool()); + if let Some(ref db) = db { + let mut workspace = Workspace::new_with_db("default", Arc::clone(db)); if let Some(ref emb) = embeddings { workspace = workspace.with_embeddings(emb.clone()); } @@ -445,20 +520,46 @@ async fn main() -> anyhow::Result<()> { tracing::info!("Builder mode enabled"); } - // Create secrets store if master key is configured (needed for MCP auth and WASM channels) + // Create secrets store if master key is configured (needed for MCP auth and WASM channels). + // + // When both `postgres` and `libsql` features are compiled, the runtime-selected + // backend determines which store is created: whichever DB init branch ran will + // have set its handle (pg_pool or libsql_db), and the or_else chain picks it up. let secrets_store: Option> = - if let (Some(store), Some(master_key)) = (&store, config.secrets.master_key()) { + if let Some(master_key) = config.secrets.master_key() { match SecretsCrypto::new(master_key.clone()) { - Ok(crypto) => Some(Arc::new(PostgresSecretsStore::new( - store.pool(), - Arc::new(crypto), - ))), + Ok(crypto) => { + let crypto = Arc::new(crypto); + let store: Option> = None; + + #[cfg(feature = "libsql")] + let store = store.or_else(|| { + libsql_db.take().map(|db| { + Arc::new(LibSqlSecretsStore::new(db, Arc::clone(&crypto))) + as Arc + }) + }); + + #[cfg(feature = "postgres")] + let store = store.or_else(|| { + pg_pool.as_ref().map(|pool| { + Arc::new(PostgresSecretsStore::new(pool.clone(), Arc::clone(&crypto))) + as Arc + }) + }); + + store + } Err(e) => { tracing::warn!("Failed to initialize secrets crypto: {}", e); + #[cfg(feature = "libsql")] + let _ = libsql_db.take(); None } } } else { + #[cfg(feature = "libsql")] + let _ = libsql_db.take(); None }; @@ -522,8 +623,8 @@ async fn main() -> anyhow::Result<()> { let mcp_servers_future = async { if let Some(ref secrets) = secrets_store { - let servers_result = if let Some(ref s) = store { - load_mcp_servers_from_db(s, "default").await + let servers_result = if let Some(ref d) = db { + load_mcp_servers_from_db(d.as_ref(), "default").await } else { ironclaw::tools::mcp::config::load_mcp_servers().await }; @@ -636,7 +737,7 @@ async fn main() -> anyhow::Result<()> { config.channels.wasm_channels_dir.clone(), config.tunnel.public_url.clone(), "default".to_string(), - store.clone(), + db.clone(), )); tools.register_extension_tools(Arc::clone(&manager)); tracing::info!("Extension manager initialized with in-chat discovery tools"); @@ -699,7 +800,7 @@ async fn main() -> anyhow::Result<()> { token_store, job_event_tx: job_event_tx.clone(), prompt_queue: Arc::clone(&prompt_queue), - store: store.clone(), + store: db.clone(), }; tokio::spawn(async move { @@ -936,13 +1037,15 @@ async fn main() -> anyhow::Result<()> { }; // Create workspace for agent (shared with memory tools) - let workspace = store.as_ref().map(|s| { - let mut ws = Workspace::new("default", s.pool()); + let workspace = if let Some(ref db_ref) = db { + let mut ws = Workspace::new_with_db("default", Arc::clone(db_ref)); if let Some(ref emb) = embeddings { ws = ws.with_embeddings(emb.clone()); } - Arc::new(ws) - }); + Some(Arc::new(ws)) + } else { + None + }; // Seed workspace with core identity files on first boot if let Some(ref ws) = workspace { @@ -980,7 +1083,7 @@ async fn main() -> anyhow::Result<()> { tools.register_job_tools( Arc::clone(&context_manager), container_job_manager.clone(), - store.clone(), + db.clone(), ); // Add web gateway channel if configured @@ -995,13 +1098,12 @@ async fn main() -> anyhow::Result<()> { if let Some(ref ext_mgr) = extension_manager { gw = gw.with_extension_manager(Arc::clone(ext_mgr)); } - if let Some(ref s) = store { - gw = gw.with_store(Arc::clone(s)); + if let Some(ref d) = db { + gw = gw.with_store(Arc::clone(d)); } if let Some(ref jm) = container_job_manager { gw = gw.with_job_manager(Arc::clone(jm)); } - gw = gw.with_llm_provider(Arc::clone(&llm)); if config.sandbox.enabled { gw = gw.with_prompt_queue(Arc::clone(&prompt_queue)); @@ -1034,7 +1136,7 @@ async fn main() -> anyhow::Result<()> { // Create and run the agent let deps = AgentDeps { - store, + store: db, llm, safety, tools, @@ -1068,11 +1170,17 @@ async fn main() -> anyhow::Result<()> { /// Check if onboarding is needed and return the reason. /// /// Returns `Some(reason)` if onboarding should be triggered, `None` otherwise. +#[cfg(any(feature = "postgres", feature = "libsql"))] async fn check_onboard_needed() -> Option<&'static str> { let bootstrap = ironclaw::bootstrap::BootstrapConfig::load(); // Database not configured (and not in env) - if bootstrap.database_url.is_none() && std::env::var("DATABASE_URL").is_err() { + let has_db = bootstrap.database_url.is_some() + || std::env::var("DATABASE_URL").is_ok() + || std::env::var("LIBSQL_PATH").is_ok() + || ironclaw::config::default_libsql_path().exists(); + + if !has_db { return Some("Database not configured"); } diff --git a/src/orchestrator/api.rs b/src/orchestrator/api.rs index 8923bc55..012ec270 100644 --- a/src/orchestrator/api.rs +++ b/src/orchestrator/api.rs @@ -15,7 +15,7 @@ use tokio::sync::{Mutex, broadcast}; use uuid::Uuid; use crate::channels::web::types::SseEvent; -use crate::history::Store; +use crate::db::Database; use crate::llm::{CompletionRequest, LlmProvider, ToolCompletionRequest}; use crate::orchestrator::auth::{TokenStore, worker_auth_middleware}; use crate::orchestrator::job_manager::ContainerJobManager; @@ -43,7 +43,7 @@ pub struct OrchestratorState { /// Buffered follow-up prompts for sandbox jobs, keyed by job_id. pub prompt_queue: Arc>>>, /// Database handle for persisting job events. - pub store: Option>, + pub store: Option>, } /// The orchestrator's internal API server. diff --git a/src/secrets/mod.rs b/src/secrets/mod.rs index d2f8696d..8547fbfc 100644 --- a/src/secrets/mod.rs +++ b/src/secrets/mod.rs @@ -64,7 +64,11 @@ mod store; mod types; pub use crypto::SecretsCrypto; -pub use store::{PostgresSecretsStore, SecretsStore}; +#[cfg(feature = "libsql")] +pub use store::LibSqlSecretsStore; +#[cfg(feature = "postgres")] +pub use store::PostgresSecretsStore; +pub use store::SecretsStore; pub use types::{ CreateSecretParams, CredentialLocation, CredentialMapping, DecryptedSecret, Secret, SecretError, SecretRef, diff --git a/src/secrets/store.rs b/src/secrets/store.rs index 442bbcb9..cf532a01 100644 --- a/src/secrets/store.rs +++ b/src/secrets/store.rs @@ -10,6 +10,7 @@ use std::sync::Arc; use async_trait::async_trait; use chrono::Utc; +#[cfg(feature = "postgres")] use deadpool_postgres::Pool; use secrecy::ExposeSecret; use uuid::Uuid; @@ -61,11 +62,13 @@ pub trait SecretsStore: Send + Sync { } /// PostgreSQL implementation of SecretsStore. +#[cfg(feature = "postgres")] pub struct PostgresSecretsStore { pool: Pool, crypto: Arc, } +#[cfg(feature = "postgres")] impl PostgresSecretsStore { /// Create a new store with the given database pool and crypto instance. pub fn new(pool: Pool, crypto: Arc) -> Self { @@ -73,6 +76,7 @@ impl PostgresSecretsStore { } } +#[cfg(feature = "postgres")] #[async_trait] impl SecretsStore for PostgresSecretsStore { async fn create( @@ -283,6 +287,7 @@ impl SecretsStore for PostgresSecretsStore { } } +#[cfg(feature = "postgres")] fn row_to_secret(row: &tokio_postgres::Row) -> Secret { Secret { id: row.get("id"), @@ -299,6 +304,332 @@ fn row_to_secret(row: &tokio_postgres::Row) -> Secret { } } +// ==================== libSQL implementation ==================== + +/// libSQL/Turso implementation of SecretsStore. +/// +/// Holds an `Arc` handle and creates a fresh connection per operation, +/// matching the connection-per-request pattern used by the main `LibSqlBackend`. +#[cfg(feature = "libsql")] +pub struct LibSqlSecretsStore { + db: Arc, + crypto: Arc, +} + +#[cfg(feature = "libsql")] +impl LibSqlSecretsStore { + /// Create a new store with the given shared libsql database handle and crypto instance. + pub fn new(db: Arc, crypto: Arc) -> Self { + Self { db, crypto } + } + + fn connect(&self) -> Result { + self.db + .connect() + .map_err(|e| SecretError::Database(format!("Connection failed: {}", e))) + } +} + +#[cfg(feature = "libsql")] +#[async_trait] +impl SecretsStore for LibSqlSecretsStore { + async fn create( + &self, + user_id: &str, + params: CreateSecretParams, + ) -> Result { + let plaintext = params.value.expose_secret().as_bytes(); + let (encrypted_value, key_salt) = self.crypto.encrypt(plaintext)?; + + let id = Uuid::new_v4(); + let now = Utc::now(); + let now_str = now.to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + let expires_at_str = params + .expires_at + .map(|dt| dt.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)); + + // Start transaction for atomic upsert + read-back + let conn = self.connect()?; + let tx = conn + .transaction() + .await + .map_err(|e| SecretError::Database(e.to_string()))?; + + tx.execute( + r#" + INSERT INTO secrets (id, user_id, name, encrypted_value, key_salt, provider, expires_at, created_at, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?8) + ON CONFLICT (user_id, name) DO UPDATE SET + encrypted_value = excluded.encrypted_value, + key_salt = excluded.key_salt, + provider = excluded.provider, + expires_at = excluded.expires_at, + updated_at = ?8 + "#, + libsql::params![ + id.to_string(), + user_id, + params.name.as_str(), + libsql::Value::Blob(encrypted_value.clone()), + libsql::Value::Blob(key_salt.clone()), + libsql_opt_text(params.provider.as_deref()), + libsql_opt_text(expires_at_str.as_deref()), + now_str.as_str(), + ], + ) + .await + .map_err(|e| SecretError::Database(e.to_string()))?; + + // Read back the row (may have been upserted) + let mut rows = tx + .query( + r#" + SELECT id, user_id, name, encrypted_value, key_salt, provider, expires_at, + last_used_at, usage_count, created_at, updated_at + FROM secrets + WHERE user_id = ?1 AND name = ?2 + "#, + libsql::params![user_id, params.name.as_str()], + ) + .await + .map_err(|e| SecretError::Database(e.to_string()))?; + + let row = rows + .next() + .await + .map_err(|e| SecretError::Database(e.to_string()))? + .ok_or_else(|| SecretError::Database("Insert succeeded but row not found".into()))?; + + let secret = libsql_row_to_secret(&row)?; + + tx.commit() + .await + .map_err(|e| SecretError::Database(e.to_string()))?; + + Ok(secret) + } + + async fn get(&self, user_id: &str, name: &str) -> Result { + let conn = self.connect()?; + let mut rows = conn + .query( + r#" + SELECT id, user_id, name, encrypted_value, key_salt, provider, expires_at, + last_used_at, usage_count, created_at, updated_at + FROM secrets + WHERE user_id = ?1 AND name = ?2 + "#, + libsql::params![user_id, name], + ) + .await + .map_err(|e| SecretError::Database(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| SecretError::Database(e.to_string()))? + { + Some(row) => { + let secret = libsql_row_to_secret(&row)?; + + if let Some(expires_at) = secret.expires_at + && expires_at < Utc::now() + { + return Err(SecretError::Expired); + } + + Ok(secret) + } + None => Err(SecretError::NotFound(name.to_string())), + } + } + + async fn get_decrypted( + &self, + user_id: &str, + name: &str, + ) -> Result { + let secret = self.get(user_id, name).await?; + self.crypto + .decrypt(&secret.encrypted_value, &secret.key_salt) + } + + async fn exists(&self, user_id: &str, name: &str) -> Result { + let conn = self.connect()?; + let mut rows = conn + .query( + "SELECT 1 FROM secrets WHERE user_id = ?1 AND name = ?2", + libsql::params![user_id, name], + ) + .await + .map_err(|e| SecretError::Database(e.to_string()))?; + + Ok(rows + .next() + .await + .map_err(|e| SecretError::Database(e.to_string()))? + .is_some()) + } + + async fn list(&self, user_id: &str) -> Result, SecretError> { + let conn = self.connect()?; + let mut rows = conn + .query( + "SELECT name, provider FROM secrets WHERE user_id = ?1 ORDER BY name", + libsql::params![user_id], + ) + .await + .map_err(|e| SecretError::Database(e.to_string()))?; + + let mut refs = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| SecretError::Database(e.to_string()))? + { + refs.push(SecretRef { + name: row.get::(0).unwrap_or_default(), + provider: row.get::(1).ok(), + }); + } + Ok(refs) + } + + async fn delete(&self, user_id: &str, name: &str) -> Result { + let conn = self.connect()?; + let affected = conn + .execute( + "DELETE FROM secrets WHERE user_id = ?1 AND name = ?2", + libsql::params![user_id, name], + ) + .await + .map_err(|e| SecretError::Database(e.to_string()))?; + + Ok(affected > 0) + } + + async fn record_usage(&self, secret_id: Uuid) -> Result<(), SecretError> { + let now = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + let conn = self.connect()?; + + conn.execute( + r#" + UPDATE secrets + SET last_used_at = ?1, usage_count = usage_count + 1 + WHERE id = ?2 + "#, + libsql::params![now.as_str(), secret_id.to_string()], + ) + .await + .map_err(|e| SecretError::Database(e.to_string()))?; + + Ok(()) + } + + async fn is_accessible( + &self, + user_id: &str, + secret_name: &str, + allowed_secrets: &[String], + ) -> Result { + if !self.exists(user_id, secret_name).await? { + return Ok(false); + } + + for pattern in allowed_secrets { + if pattern == secret_name { + return Ok(true); + } + + if let Some(prefix) = pattern.strip_suffix('*') + && secret_name.starts_with(prefix) + { + return Ok(true); + } + } + + Ok(false) + } +} + +#[cfg(feature = "libsql")] +fn libsql_opt_text(s: Option<&str>) -> libsql::Value { + match s { + Some(s) => libsql::Value::Text(s.to_string()), + None => libsql::Value::Null, + } +} + +#[cfg(feature = "libsql")] +fn libsql_parse_timestamp(s: &str) -> Result, SecretError> { + if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(s) { + return Ok(dt.with_timezone(&Utc)); + } + if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f") { + return Ok(ndt.and_utc()); + } + if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S") { + return Ok(ndt.and_utc()); + } + Err(SecretError::Database(format!( + "unparseable timestamp: {:?}", + s + ))) +} + +#[cfg(feature = "libsql")] +fn libsql_row_to_secret(row: &libsql::Row) -> Result { + let id_str: String = row + .get(0) + .map_err(|e| SecretError::Database(e.to_string()))?; + let user_id: String = row + .get(1) + .map_err(|e| SecretError::Database(e.to_string()))?; + let name: String = row + .get(2) + .map_err(|e| SecretError::Database(e.to_string()))?; + let encrypted_value: Vec = row + .get(3) + .map_err(|e| SecretError::Database(e.to_string()))?; + let key_salt: Vec = row + .get(4) + .map_err(|e| SecretError::Database(e.to_string()))?; + let provider: Option = row.get::(5).ok().filter(|s| !s.is_empty()); + let expires_at = row + .get::(6) + .ok() + .filter(|s| !s.is_empty()) + .and_then(|s| libsql_parse_timestamp(&s).ok()); + let last_used_at = row + .get::(7) + .ok() + .filter(|s| !s.is_empty()) + .and_then(|s| libsql_parse_timestamp(&s).ok()); + let usage_count: i64 = row.get::(8).unwrap_or(0); + let created_at_str: String = row + .get(9) + .map_err(|e| SecretError::Database(e.to_string()))?; + let updated_at_str: String = row + .get(10) + .map_err(|e| SecretError::Database(e.to_string()))?; + + Ok(Secret { + id: id_str + .parse() + .map_err(|e: uuid::Error| SecretError::Database(e.to_string()))?, + user_id, + name, + encrypted_value, + key_salt, + provider, + expires_at, + last_used_at, + usage_count, + created_at: libsql_parse_timestamp(&created_at_str)?, + updated_at: libsql_parse_timestamp(&updated_at_str)?, + }) +} + /// In-memory implementation for testing. #[cfg(test)] pub mod testing { diff --git a/src/settings.rs b/src/settings.rs index 59165c55..08cc6b6a 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -15,6 +15,10 @@ pub struct Settings { pub onboard_completed: bool, // === Step 1: Database === + /// Database backend: "postgres" or "libsql". + #[serde(default)] + pub database_backend: Option, + /// Database connection URL (postgres://...). #[serde(default)] pub database_url: Option, @@ -23,6 +27,14 @@ pub struct Settings { #[serde(default)] pub database_pool_size: Option, + /// Path to local libSQL database file. + #[serde(default)] + pub libsql_path: Option, + + /// Turso cloud URL for remote replica sync. + #[serde(default)] + pub libsql_url: Option, + // === Step 2: Security === /// Source for the secrets master key. #[serde(default)] diff --git a/src/setup/channels.rs b/src/setup/channels.rs index 3de5cb23..77fcc38d 100644 --- a/src/setup/channels.rs +++ b/src/setup/channels.rs @@ -12,7 +12,9 @@ use reqwest::Client; use secrecy::{ExposeSecret, SecretString}; use serde::Deserialize; -use crate::secrets::{CreateSecretParams, PostgresSecretsStore, SecretsCrypto, SecretsStore}; +#[cfg(feature = "postgres")] +use crate::secrets::SecretsCrypto; +use crate::secrets::{CreateSecretParams, SecretsStore}; use crate::settings::Settings; use crate::setup::prompts::{ confirm, input, optional_input, print_error, print_info, print_success, secret_input, @@ -20,15 +22,24 @@ use crate::setup::prompts::{ /// Context for saving secrets during setup. pub struct SecretsContext { - store: PostgresSecretsStore, + store: Arc, user_id: String, } impl SecretsContext { - /// Create a new secrets context. + /// Create a new secrets context from a trait-object store. + pub fn from_store(store: Arc, user_id: &str) -> Self { + Self { + store, + user_id: user_id.to_string(), + } + } + + /// Create a new secrets context from a PostgreSQL pool and crypto. + #[cfg(feature = "postgres")] pub fn new(pool: deadpool_postgres::Pool, crypto: Arc, user_id: &str) -> Self { Self { - store: PostgresSecretsStore::new(pool, crypto), + store: Arc::new(crate::secrets::PostgresSecretsStore::new(pool, crypto)), user_id: user_id.to_string(), } } diff --git a/src/setup/mod.rs b/src/setup/mod.rs index 71bda14a..ca4d4c56 100644 --- a/src/setup/mod.rs +++ b/src/setup/mod.rs @@ -20,6 +20,7 @@ mod channels; mod prompts; +#[cfg(any(feature = "postgres", feature = "libsql"))] mod wizard; pub use channels::{ @@ -29,4 +30,5 @@ pub use prompts::{ confirm, input, optional_input, print_error, print_header, print_info, print_step, print_success, secret_input, select_many, select_one, }; +#[cfg(any(feature = "postgres", feature = "libsql"))] pub use wizard::{SetupConfig, SetupWizard}; diff --git a/src/setup/wizard.rs b/src/setup/wizard.rs index 632d4272..696a7a08 100644 --- a/src/setup/wizard.rs +++ b/src/setup/wizard.rs @@ -12,15 +12,17 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; +#[cfg(feature = "postgres")] use deadpool_postgres::{Config as PoolConfig, Runtime}; use secrecy::SecretString; +#[cfg(feature = "postgres")] use tokio_postgres::NoTls; use crate::channels::wasm::{ ChannelCapabilitiesFile, available_channel_names, install_bundled_channel, }; use crate::llm::{SessionConfig, SessionManager}; -use crate::secrets::SecretsCrypto; +use crate::secrets::{SecretsCrypto, SecretsStore}; use crate::settings::{KeySource, Settings}; use crate::setup::channels::{ SecretsContext, setup_http, setup_telegram, setup_tunnel, setup_wasm_channel, @@ -66,8 +68,12 @@ pub struct SetupWizard { config: SetupConfig, settings: Settings, session_manager: Option>, - /// Database pool (created during setup). + /// Database pool (created during setup, postgres only). + #[cfg(feature = "postgres")] db_pool: Option, + /// libSQL backend (created during setup, libsql only). + #[cfg(feature = "libsql")] + db_backend: Option, /// Secrets crypto (created during setup). secrets_crypto: Option>, } @@ -79,7 +85,10 @@ impl SetupWizard { config: SetupConfig::default(), settings: Settings::load(), session_manager: None, + #[cfg(feature = "postgres")] db_pool: None, + #[cfg(feature = "libsql")] + db_backend: None, secrets_crypto: None, } } @@ -90,7 +99,10 @@ impl SetupWizard { config, settings: Settings::load(), session_manager: None, + #[cfg(feature = "postgres")] db_pool: None, + #[cfg(feature = "libsql")] + db_backend: None, secrets_crypto: None, } } @@ -153,19 +165,48 @@ impl SetupWizard { /// Step 1: Database connection. async fn step_database(&mut self) -> Result<(), SetupError> { - // Check if we have an existing URL in env or settings + // Determine which backend to use based on compile-time features. + // When both features are enabled, prefer the currently configured backend + // or default to postgres. + #[cfg(all(feature = "postgres", feature = "libsql"))] + { + let backend = std::env::var("DATABASE_BACKEND") + .ok() + .or_else(|| self.settings.database_backend.clone()) + .unwrap_or_else(|| "postgres".to_string()); + + if backend == "libsql" || backend == "turso" || backend == "sqlite" { + return self.step_database_libsql().await; + } + return self.step_database_postgres().await; + } + + #[cfg(all(feature = "postgres", not(feature = "libsql")))] + { + return self.step_database_postgres().await; + } + + #[cfg(all(feature = "libsql", not(feature = "postgres")))] + { + return self.step_database_libsql().await; + } + } + + /// Step 1 (postgres): Database connection via PostgreSQL URL. + #[cfg(feature = "postgres")] + async fn step_database_postgres(&mut self) -> Result<(), SetupError> { + self.settings.database_backend = Some("postgres".to_string()); + let existing_url = std::env::var("DATABASE_URL") .ok() .or_else(|| self.settings.database_url.clone()); if let Some(ref url) = existing_url { - // Mask the password for display let display_url = mask_password_in_url(url); print_info(&format!("Existing database URL: {}", display_url)); if confirm("Use this database?", true).map_err(SetupError::Io)? { - // Test the connection - if let Err(e) = self.test_database_connection(url).await { + if let Err(e) = self.test_database_connection_postgres(url).await { print_error(&format!("Connection failed: {}", e)); print_info("Let's configure a new database URL."); } else { @@ -176,7 +217,6 @@ impl SetupWizard { } } - // Prompt for new URL println!(); print_info("Enter your PostgreSQL connection URL."); print_info("Format: postgres://user:password@host:port/database"); @@ -190,15 +230,13 @@ impl SetupWizard { continue; } - // Test the connection print_info("Testing connection..."); - match self.test_database_connection(&url).await { + match self.test_database_connection_postgres(&url).await { Ok(()) => { print_success("Database connection successful"); - // Ask if we should run migrations if confirm("Run database migrations?", true).map_err(SetupError::Io)? { - self.run_migrations().await?; + self.run_migrations_postgres().await?; } self.settings.database_url = Some(url); @@ -216,8 +254,115 @@ impl SetupWizard { } } - /// Test database connection and store the pool. - async fn test_database_connection(&mut self, url: &str) -> Result<(), SetupError> { + /// Step 1 (libsql): Database connection via local file or Turso remote replica. + #[cfg(feature = "libsql")] + async fn step_database_libsql(&mut self) -> Result<(), SetupError> { + self.settings.database_backend = Some("libsql".to_string()); + + let default_path = crate::config::default_libsql_path(); + let default_path_str = default_path.to_string_lossy().to_string(); + + // Check for existing configuration + let existing_path = std::env::var("LIBSQL_PATH") + .ok() + .or_else(|| self.settings.libsql_path.clone()); + + if let Some(ref path) = existing_path { + print_info(&format!("Existing database path: {}", path)); + if confirm("Use this database?", true).map_err(SetupError::Io)? { + let turso_url = std::env::var("LIBSQL_URL") + .ok() + .or_else(|| self.settings.libsql_url.clone()); + let turso_token = std::env::var("LIBSQL_AUTH_TOKEN").ok(); + + match self + .test_database_connection_libsql( + path, + turso_url.as_deref(), + turso_token.as_deref(), + ) + .await + { + Ok(()) => { + print_success("Database connection successful"); + self.settings.libsql_path = Some(path.clone()); + if let Some(url) = turso_url { + self.settings.libsql_url = Some(url); + } + return Ok(()); + } + Err(e) => { + print_error(&format!("Connection failed: {}", e)); + print_info("Let's configure a new database path."); + } + } + } + } + + println!(); + print_info("IronClaw uses an embedded SQLite database (libSQL)."); + print_info("No external database server required."); + println!(); + + let path_input = optional_input( + "Database file path", + Some(&format!("default: {}", default_path_str)), + ) + .map_err(SetupError::Io)?; + + let db_path = path_input.unwrap_or(default_path_str.clone()); + + // Ask about Turso cloud sync + println!(); + let use_turso = + confirm("Enable Turso cloud sync (remote replica)?", false).map_err(SetupError::Io)?; + + let (turso_url, turso_token) = if use_turso { + print_info("Enter your Turso database URL and auth token."); + print_info("Format: libsql://your-db.turso.io"); + println!(); + + let url = input("Turso URL").map_err(SetupError::Io)?; + if url.is_empty() { + print_error("Turso URL is required for cloud sync."); + (None, None) + } else { + let token = input("Auth token").map_err(SetupError::Io)?; + if token.is_empty() { + print_error("Auth token is required for cloud sync."); + (None, None) + } else { + (Some(url), Some(token)) + } + } + } else { + (None, None) + }; + + print_info("Testing connection..."); + match self + .test_database_connection_libsql(&db_path, turso_url.as_deref(), turso_token.as_deref()) + .await + { + Ok(()) => { + print_success("Database connection successful"); + + // Always run migrations for libsql (they're idempotent) + self.run_migrations_libsql().await?; + + self.settings.libsql_path = Some(db_path); + if let Some(url) = turso_url { + self.settings.libsql_url = Some(url); + } + Ok(()) + } + Err(e) => Err(SetupError::Database(format!("Connection failed: {}", e))), + } + } + + /// Test PostgreSQL connection and store the pool. + #[cfg(feature = "postgres")] + async fn test_database_connection_postgres(&mut self, url: &str) -> Result<(), SetupError> { let mut cfg = PoolConfig::new(); cfg.url = Some(url.to_string()); cfg.pool = Some(deadpool_postgres::PoolConfig { @@ -229,7 +374,6 @@ impl SetupWizard { .create_pool(Some(Runtime::Tokio1), NoTls) .map_err(|e| SetupError::Database(format!("Failed to create pool: {}", e)))?; - // Test the connection let _ = pool .get() .await @@ -239,8 +383,36 @@ impl SetupWizard { Ok(()) } - /// Run database migrations. - async fn run_migrations(&self) -> Result<(), SetupError> { + /// Test libSQL connection and store the backend. + #[cfg(feature = "libsql")] + async fn test_database_connection_libsql( + &mut self, + path: &str, + turso_url: Option<&str>, + turso_token: Option<&str>, + ) -> Result<(), SetupError> { + use crate::db::libsql_backend::LibSqlBackend; + use std::path::Path; + + let db_path = Path::new(path); + + let backend = if let (Some(url), Some(token)) = (turso_url, turso_token) { + LibSqlBackend::new_remote_replica(db_path, url, token) + .await + .map_err(|e| SetupError::Database(format!("Failed to connect: {}", e)))? + } else { + LibSqlBackend::new_local(db_path) + .await + .map_err(|e| SetupError::Database(format!("Failed to open database: {}", e)))? + }; + + self.db_backend = Some(backend); + Ok(()) + } + + /// Run PostgreSQL migrations. + #[cfg(feature = "postgres")] + async fn run_migrations_postgres(&self) -> Result<(), SetupError> { if let Some(ref pool) = self.db_pool { use refinery::embed_migrations; embed_migrations!("migrations"); @@ -262,6 +434,24 @@ impl SetupWizard { Ok(()) } + /// Run libSQL migrations. + #[cfg(feature = "libsql")] + async fn run_migrations_libsql(&self) -> Result<(), SetupError> { + if let Some(ref backend) = self.db_backend { + use crate::db::Database; + + print_info("Running migrations..."); + + backend + .run_migrations() + .await + .map_err(|e| SetupError::Database(format!("Migration failed: {}", e)))?; + + print_success("Migrations applied"); + } + Ok(()) + } + /// Step 2: Security (secrets master key). async fn step_security(&mut self) -> Result<(), SetupError> { // Check current configuration @@ -532,24 +722,6 @@ impl SetupWizard { /// Initialize secrets context for channel setup. async fn init_secrets_context(&mut self) -> Result { - // Get database pool (should be set from step 1) - let pool = if let Some(ref p) = self.db_pool { - p.clone() - } else { - // Fall back to creating one from settings/env - let url = self - .settings - .database_url - .clone() - .or_else(|| std::env::var("DATABASE_URL").ok()) - .ok_or_else(|| SetupError::Config("Database URL not configured".to_string()))?; - - self.test_database_connection(&url).await?; - // Ensure secrets-related tables exist for channels-only onboarding flows. - self.run_migrations().await?; - self.db_pool.clone().unwrap() - }; - // Get crypto (should be set from step 2, or load from keychain/env) let crypto = if let Some(ref c) = self.secrets_crypto { Arc::clone(c) @@ -571,7 +743,74 @@ impl SetupWizard { Arc::clone(self.secrets_crypto.as_ref().unwrap()) }; - Ok(SecretsContext::new(pool, crypto, "default")) + // Create backend-appropriate secrets store + #[cfg(feature = "postgres")] + { + // Try postgres path first when postgres feature is available + if let Some(store) = self.create_postgres_secrets_store(&crypto).await? { + return Ok(SecretsContext::from_store(store, "default")); + } + } + + #[cfg(feature = "libsql")] + { + if let Some(store) = self.create_libsql_secrets_store(&crypto)? { + return Ok(SecretsContext::from_store(store, "default")); + } + } + + Err(SetupError::Config( + "No database backend available for secrets storage".to_string(), + )) + } + + /// Create a PostgreSQL secrets store from the current pool. + #[cfg(feature = "postgres")] + async fn create_postgres_secrets_store( + &mut self, + crypto: &Arc, + ) -> Result>, SetupError> { + let pool = if let Some(ref p) = self.db_pool { + p.clone() + } else { + // Fall back to creating one from settings/env + let url = self + .settings + .database_url + .clone() + .or_else(|| std::env::var("DATABASE_URL").ok()); + + if let Some(url) = url { + self.test_database_connection_postgres(&url).await?; + self.run_migrations_postgres().await?; + self.db_pool.clone().unwrap() + } else { + return Ok(None); + } + }; + + let store: Arc = Arc::new(crate::secrets::PostgresSecretsStore::new( + pool, + Arc::clone(crypto), + )); + Ok(Some(store)) + } + + /// Create a libSQL secrets store from the current backend. + #[cfg(feature = "libsql")] + fn create_libsql_secrets_store( + &self, + crypto: &Arc, + ) -> Result>, SetupError> { + if let Some(ref backend) = self.db_backend { + let store: Arc = Arc::new(crate::secrets::LibSqlSecretsStore::new( + backend.shared_db(), + Arc::clone(crypto), + )); + Ok(Some(store)) + } else { + Ok(None) + } } /// Step 6: Channel configuration. @@ -793,8 +1032,27 @@ impl SetupWizard { println!("Configuration Summary:"); println!("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"); - if self.settings.database_url.is_some() { - println!(" Database: configured"); + let backend = self + .settings + .database_backend + .as_deref() + .unwrap_or("postgres"); + match backend { + "libsql" => { + if let Some(ref path) = self.settings.libsql_path { + println!(" Database: libSQL ({})", path); + } else { + println!(" Database: libSQL (default path)"); + } + if self.settings.libsql_url.is_some() { + println!(" Turso sync: enabled"); + } + } + _ => { + if self.settings.database_url.is_some() { + println!(" Database: PostgreSQL (configured)"); + } + } } match self.settings.secrets_master_key_source { @@ -874,6 +1132,7 @@ impl Default for SetupWizard { } /// Mask password in a database URL for display. +#[cfg(feature = "postgres")] fn mask_password_in_url(url: &str) -> String { // URL format: scheme://user:password@host/database // Find "://" to locate start of credentials @@ -1064,6 +1323,7 @@ mod tests { } #[test] + #[cfg(feature = "postgres")] fn test_mask_password_in_url() { assert_eq!( mask_password_in_url("postgres://user:secret@localhost/db"), diff --git a/src/tools/builtin/job.rs b/src/tools/builtin/job.rs index 86552c34..015c3b13 100644 --- a/src/tools/builtin/job.rs +++ b/src/tools/builtin/job.rs @@ -15,7 +15,8 @@ use chrono::Utc; use uuid::Uuid; use crate::context::{ContextManager, JobContext, JobState}; -use crate::history::{SandboxJobRecord, Store}; +use crate::db::Database; +use crate::history::SandboxJobRecord; use crate::orchestrator::job_manager::{ContainerJobManager, JobMode}; use crate::tools::tool::{Tool, ToolError, ToolOutput}; @@ -27,7 +28,7 @@ use crate::tools::tool::{Tool, ToolError, ToolOutput}; pub struct CreateJobTool { context_manager: Arc, job_manager: Option>, - store: Option>, + store: Option>, } impl CreateJobTool { @@ -43,7 +44,7 @@ impl CreateJobTool { pub fn with_sandbox( mut self, job_manager: Arc, - store: Option>, + store: Option>, ) -> Self { self.job_manager = Some(job_manager); self.store = store; diff --git a/src/tools/builtin/memory.rs b/src/tools/builtin/memory.rs index a64e5863..8a2bbbdb 100644 --- a/src/tools/builtin/memory.rs +++ b/src/tools/builtin/memory.rs @@ -482,7 +482,7 @@ impl Tool for MemoryTreeTool { } } -#[cfg(test)] +#[cfg(all(test, feature = "postgres"))] mod tests { use super::*; diff --git a/src/tools/builtin/routine.rs b/src/tools/builtin/routine.rs index decf7357..338601bc 100644 --- a/src/tools/builtin/routine.rs +++ b/src/tools/builtin/routine.rs @@ -19,18 +19,18 @@ use crate::agent::routine::{ }; use crate::agent::routine_engine::RoutineEngine; use crate::context::JobContext; -use crate::history::Store; +use crate::db::Database; use crate::tools::tool::{Tool, ToolError, ToolOutput}; // ==================== routine_create ==================== pub struct RoutineCreateTool { - store: Arc, + store: Arc, engine: Arc, } impl RoutineCreateTool { - pub fn new(store: Arc, engine: Arc) -> Self { + pub fn new(store: Arc, engine: Arc) -> Self { Self { store, engine } } } @@ -277,11 +277,11 @@ impl Tool for RoutineCreateTool { // ==================== routine_list ==================== pub struct RoutineListTool { - store: Arc, + store: Arc, } impl RoutineListTool { - pub fn new(store: Arc) -> Self { + pub fn new(store: Arc) -> Self { Self { store } } } @@ -351,12 +351,12 @@ impl Tool for RoutineListTool { // ==================== routine_update ==================== pub struct RoutineUpdateTool { - store: Arc, + store: Arc, engine: Arc, } impl RoutineUpdateTool { - pub fn new(store: Arc, engine: Arc) -> Self { + pub fn new(store: Arc, engine: Arc) -> Self { Self { store, engine } } } @@ -474,12 +474,12 @@ impl Tool for RoutineUpdateTool { // ==================== routine_delete ==================== pub struct RoutineDeleteTool { - store: Arc, + store: Arc, engine: Arc, } impl RoutineDeleteTool { - pub fn new(store: Arc, engine: Arc) -> Self { + pub fn new(store: Arc, engine: Arc) -> Self { Self { store, engine } } } @@ -551,11 +551,11 @@ impl Tool for RoutineDeleteTool { // ==================== routine_history ==================== pub struct RoutineHistoryTool { - store: Arc, + store: Arc, } impl RoutineHistoryTool { - pub fn new(store: Arc) -> Self { + pub fn new(store: Arc) -> Self { Self { store } } } diff --git a/src/tools/mcp/config.rs b/src/tools/mcp/config.rs index f7041934..6f57d5c4 100644 --- a/src/tools/mcp/config.rs +++ b/src/tools/mcp/config.rs @@ -333,7 +333,7 @@ pub async fn get_mcp_server(name: &str) -> Result /// /// Falls back to the disk file if DB has no entry. pub async fn load_mcp_servers_from_db( - store: &crate::history::Store, + store: &dyn crate::db::Database, user_id: &str, ) -> Result { match store.get_setting(user_id, "mcp_servers").await { @@ -357,7 +357,7 @@ pub async fn load_mcp_servers_from_db( /// Save MCP server configurations to the database settings table. pub async fn save_mcp_servers_to_db( - store: &crate::history::Store, + store: &dyn crate::db::Database, user_id: &str, config: &McpServersFile, ) -> Result<(), ConfigError> { @@ -371,7 +371,7 @@ pub async fn save_mcp_servers_to_db( /// Add a new MCP server configuration (DB-backed). pub async fn add_mcp_server_db( - store: &crate::history::Store, + store: &dyn crate::db::Database, user_id: &str, config: McpServerConfig, ) -> Result<(), ConfigError> { @@ -386,7 +386,7 @@ pub async fn add_mcp_server_db( /// Remove an MCP server by name (DB-backed). pub async fn remove_mcp_server_db( - store: &crate::history::Store, + store: &dyn crate::db::Database, user_id: &str, name: &str, ) -> Result<(), ConfigError> { diff --git a/src/tools/registry.rs b/src/tools/registry.rs index 906e048d..ae28ce5b 100644 --- a/src/tools/registry.rs +++ b/src/tools/registry.rs @@ -6,8 +6,8 @@ use std::sync::Arc; use tokio::sync::RwLock; use crate::context::ContextManager; +use crate::db::Database; use crate::extensions::ExtensionManager; -use crate::history::Store; use crate::llm::{LlmProvider, ToolDefinition}; use crate::orchestrator::job_manager::ContainerJobManager; use crate::safety::SafetyLayer; @@ -243,7 +243,7 @@ impl ToolRegistry { &self, context_manager: Arc, job_manager: Option>, - store: Option>, + store: Option>, ) { let mut create_tool = CreateJobTool::new(Arc::clone(&context_manager)); if let Some(jm) = job_manager { @@ -276,7 +276,7 @@ impl ToolRegistry { /// of routines (scheduled and event-driven tasks). pub fn register_routine_tools( &self, - store: Arc, + store: Arc, engine: Arc, ) { use crate::tools::builtin::{ diff --git a/src/tools/wasm/mod.rs b/src/tools/wasm/mod.rs index fea18e44..c80c511e 100644 --- a/src/tools/wasm/mod.rs +++ b/src/tools/wasm/mod.rs @@ -108,10 +108,13 @@ pub use credential_injector::{CredentialInjector, InjectedCredentials, Injection pub use rate_limiter::{LimitType, RateLimitError, RateLimitResult, RateLimiter}; // Storage (V2) +#[cfg(feature = "libsql")] +pub use storage::LibSqlWasmToolStore; +#[cfg(feature = "postgres")] +pub use storage::PostgresWasmToolStore; pub use storage::{ - PostgresWasmToolStore, StoreToolParams, StoredCapabilities, StoredWasmTool, - StoredWasmToolWithBinary, ToolStatus, TrustLevel, WasmStorageError, WasmToolStore, - compute_binary_hash, verify_binary_integrity, + StoreToolParams, StoredCapabilities, StoredWasmTool, StoredWasmToolWithBinary, ToolStatus, + TrustLevel, WasmStorageError, WasmToolStore, compute_binary_hash, verify_binary_integrity, }; // Loader diff --git a/src/tools/wasm/storage.rs b/src/tools/wasm/storage.rs index d29a8afb..40d9857f 100644 --- a/src/tools/wasm/storage.rs +++ b/src/tools/wasm/storage.rs @@ -16,6 +16,7 @@ use std::collections::HashMap; use async_trait::async_trait; use chrono::{DateTime, Utc}; +#[cfg(feature = "postgres")] use deadpool_postgres::Pool; use uuid::Uuid; @@ -263,16 +264,19 @@ pub fn verify_binary_integrity(binary: &[u8], expected_hash: &[u8]) -> bool { } /// PostgreSQL implementation of WasmToolStore. +#[cfg(feature = "postgres")] pub struct PostgresWasmToolStore { pool: Pool, } +#[cfg(feature = "postgres")] impl PostgresWasmToolStore { pub fn new(pool: Pool) -> Self { Self { pool } } } +#[cfg(feature = "postgres")] #[async_trait] impl WasmToolStore for PostgresWasmToolStore { async fn store(&self, params: StoreToolParams) -> Result { @@ -538,6 +542,7 @@ impl WasmToolStore for PostgresWasmToolStore { } } +#[cfg(feature = "postgres")] fn row_to_tool(row: &tokio_postgres::Row) -> Result { let trust_level_str: String = row.get("trust_level"); let status_str: String = row.get("status"); @@ -559,6 +564,459 @@ fn row_to_tool(row: &tokio_postgres::Row) -> Result` handle and creates a fresh connection per operation, +/// matching the connection-per-request pattern used by the main `LibSqlBackend`. +#[cfg(feature = "libsql")] +pub struct LibSqlWasmToolStore { + db: std::sync::Arc, +} + +#[cfg(feature = "libsql")] +impl LibSqlWasmToolStore { + pub fn new(db: std::sync::Arc) -> Self { + Self { db } + } + + fn connect(&self) -> Result { + self.db + .connect() + .map_err(|e| WasmStorageError::Database(format!("Connection failed: {}", e))) + } +} + +#[cfg(feature = "libsql")] +#[async_trait] +impl WasmToolStore for LibSqlWasmToolStore { + async fn store(&self, params: StoreToolParams) -> Result { + let binary_hash = compute_binary_hash(¶ms.wasm_binary); + let id = Uuid::new_v4(); + let now = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + let schema_str = serde_json::to_string(¶ms.parameters_schema) + .map_err(|e| WasmStorageError::InvalidData(e.to_string()))?; + + // Wrap INSERT + read-back in a transaction to prevent TOCTOU races + let conn = self.connect()?; + let tx = conn + .transaction() + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + + tx.execute( + r#" + INSERT INTO wasm_tools ( + id, user_id, name, version, description, wasm_binary, binary_hash, + parameters_schema, source_url, trust_level, status, created_at, updated_at + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 'active', ?11, ?11) + ON CONFLICT (user_id, name, version) DO UPDATE SET + description = excluded.description, + wasm_binary = excluded.wasm_binary, + binary_hash = excluded.binary_hash, + parameters_schema = excluded.parameters_schema, + source_url = excluded.source_url, + updated_at = ?11 + "#, + libsql::params![ + id.to_string(), + params.user_id.as_str(), + params.name.as_str(), + params.version.as_str(), + params.description.as_str(), + libsql::Value::Blob(params.wasm_binary), + libsql::Value::Blob(binary_hash), + schema_str.as_str(), + libsql_wasm_opt_text(params.source_url.as_deref()), + params.trust_level.to_string(), + now.as_str(), + ], + ) + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + + // Read back the row within the same transaction + let mut rows = tx + .query( + r#" + SELECT id, user_id, name, version, description, parameters_schema, + source_url, trust_level, status, created_at, updated_at + FROM wasm_tools + WHERE user_id = ?1 AND name = ?2 + ORDER BY version DESC + LIMIT 1 + "#, + libsql::params![params.user_id.as_str(), params.name.as_str()], + ) + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + + let row = rows + .next() + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))? + .ok_or_else(|| { + WasmStorageError::Database("Insert succeeded but row not found".into()) + })?; + + let tool = libsql_row_to_tool(&row)?; + + tx.commit() + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + + Ok(tool) + } + + async fn get(&self, user_id: &str, name: &str) -> Result { + let conn = self.connect()?; + let mut rows = conn + .query( + r#" + SELECT id, user_id, name, version, description, parameters_schema, + source_url, trust_level, status, created_at, updated_at + FROM wasm_tools + WHERE user_id = ?1 AND name = ?2 AND status = 'active' + ORDER BY version DESC + LIMIT 1 + "#, + libsql::params![user_id, name], + ) + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))? + { + Some(row) => { + let tool = libsql_row_to_tool(&row)?; + match tool.status { + ToolStatus::Active => Ok(tool), + ToolStatus::Disabled => Err(WasmStorageError::Disabled), + ToolStatus::Quarantined => Err(WasmStorageError::Quarantined), + } + } + None => Err(WasmStorageError::NotFound(name.to_string())), + } + } + + async fn get_with_binary( + &self, + user_id: &str, + name: &str, + ) -> Result { + let conn = self.connect()?; + let mut rows = conn + .query( + r#" + SELECT id, user_id, name, version, description, wasm_binary, binary_hash, + parameters_schema, source_url, trust_level, status, created_at, updated_at + FROM wasm_tools + WHERE user_id = ?1 AND name = ?2 AND status = 'active' + ORDER BY version DESC + LIMIT 1 + "#, + libsql::params![user_id, name], + ) + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))? + { + Some(row) => { + let wasm_binary: Vec = row + .get(5) + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + let binary_hash: Vec = row + .get(6) + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + + if !verify_binary_integrity(&wasm_binary, &binary_hash) { + tracing::error!( + user_id = user_id, + name = name, + "WASM binary integrity check failed" + ); + return Err(WasmStorageError::IntegrityCheckFailed); + } + + // Parse metadata from the row (different column offsets due to binary/hash) + let tool = libsql_row_to_tool_with_offset(&row)?; + + match tool.status { + ToolStatus::Active => Ok(StoredWasmToolWithBinary { + tool, + wasm_binary, + binary_hash, + }), + ToolStatus::Disabled => Err(WasmStorageError::Disabled), + ToolStatus::Quarantined => Err(WasmStorageError::Quarantined), + } + } + None => Err(WasmStorageError::NotFound(name.to_string())), + } + } + + async fn get_capabilities( + &self, + tool_id: Uuid, + ) -> Result, WasmStorageError> { + let conn = self.connect()?; + let mut rows = conn + .query( + r#" + SELECT id, wasm_tool_id, http_allowlist, allowed_secrets, tool_aliases, + requests_per_minute, requests_per_hour, max_request_body_bytes, + max_response_body_bytes, workspace_read_prefixes, http_timeout_secs + FROM tool_capabilities + WHERE wasm_tool_id = ?1 + "#, + libsql::params![tool_id.to_string()], + ) + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + + match rows + .next() + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))? + { + Some(row) => { + let id_str: String = row + .get(0) + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + let tool_id_str: String = row + .get(1) + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + let http_allowlist_str: String = row.get::(2).unwrap_or_default(); + let allowed_secrets_str: String = row.get::(3).unwrap_or_default(); + let tool_aliases_str: String = row.get::(4).unwrap_or_default(); + let rpm: i64 = row.get::(5).unwrap_or(60); + let rph: i64 = row.get::(6).unwrap_or(1000); + let max_req: i64 = row.get::(7).unwrap_or(1048576); + let max_resp: i64 = row.get::(8).unwrap_or(10485760); + let ws_prefixes_str: String = row.get::(9).unwrap_or_default(); + let timeout: i64 = row.get::(10).unwrap_or(30); + + let http_allowlist: Vec = + serde_json::from_str(&http_allowlist_str).unwrap_or_default(); + let allowed_secrets: Vec = + serde_json::from_str(&allowed_secrets_str).unwrap_or_default(); + let tool_aliases: HashMap = + serde_json::from_str(&tool_aliases_str).unwrap_or_default(); + let workspace_read_prefixes: Vec = + serde_json::from_str(&ws_prefixes_str).unwrap_or_default(); + + Ok(Some(StoredCapabilities { + id: id_str + .parse() + .map_err(|e: uuid::Error| WasmStorageError::InvalidData(e.to_string()))?, + wasm_tool_id: tool_id_str + .parse() + .map_err(|e: uuid::Error| WasmStorageError::InvalidData(e.to_string()))?, + http_allowlist, + allowed_secrets, + tool_aliases, + requests_per_minute: rpm as u32, + requests_per_hour: rph as u32, + max_request_body_bytes: max_req, + max_response_body_bytes: max_resp, + workspace_read_prefixes, + http_timeout_secs: timeout as i32, + })) + } + None => Ok(None), + } + } + + async fn list(&self, user_id: &str) -> Result, WasmStorageError> { + // SQLite doesn't have DISTINCT ON, so we use a subquery to get latest version per name + let conn = self.connect()?; + let mut rows = conn + .query( + r#" + SELECT id, user_id, name, version, description, parameters_schema, + source_url, trust_level, status, created_at, updated_at + FROM wasm_tools + WHERE user_id = ?1 + AND rowid IN ( + SELECT MAX(rowid) + FROM wasm_tools + WHERE user_id = ?1 + GROUP BY name + ) + ORDER BY name + "#, + libsql::params![user_id], + ) + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + + let mut tools = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))? + { + tools.push(libsql_row_to_tool(&row)?); + } + Ok(tools) + } + + async fn update_status( + &self, + user_id: &str, + name: &str, + status: ToolStatus, + ) -> Result<(), WasmStorageError> { + let now = Utc::now().to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + let conn = self.connect()?; + + let result = conn + .execute( + "UPDATE wasm_tools SET status = ?1, updated_at = ?2 WHERE user_id = ?3 AND name = ?4", + libsql::params![status.to_string(), now.as_str(), user_id, name], + ) + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + + if result == 0 { + return Err(WasmStorageError::NotFound(name.to_string())); + } + + Ok(()) + } + + async fn delete(&self, user_id: &str, name: &str) -> Result { + let conn = self.connect()?; + let result = conn + .execute( + "DELETE FROM wasm_tools WHERE user_id = ?1 AND name = ?2", + libsql::params![user_id, name], + ) + .await + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + + Ok(result > 0) + } +} + +#[cfg(feature = "libsql")] +fn libsql_wasm_opt_text(s: Option<&str>) -> libsql::Value { + match s { + Some(s) => libsql::Value::Text(s.to_string()), + None => libsql::Value::Null, + } +} + +#[cfg(feature = "libsql")] +fn libsql_wasm_parse_ts(s: &str) -> Result, WasmStorageError> { + if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(s) { + return Ok(dt.with_timezone(&Utc)); + } + if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f") { + return Ok(ndt.and_utc()); + } + if let Ok(ndt) = chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S") { + return Ok(ndt.and_utc()); + } + Err(WasmStorageError::InvalidData(format!( + "unparseable timestamp: {:?}", + s + ))) +} + +/// Parse a tool row with standard column order (no binary columns). +/// Columns: id(0), user_id(1), name(2), version(3), description(4), +/// parameters_schema(5), source_url(6), trust_level(7), status(8), +/// created_at(9), updated_at(10) +#[cfg(feature = "libsql")] +fn libsql_row_to_tool(row: &libsql::Row) -> Result { + libsql_row_to_tool_at(row, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) +} + +/// Parse a tool row when binary columns are present (get_with_binary query). +/// Columns: id(0), user_id(1), name(2), version(3), description(4), +/// wasm_binary(5), binary_hash(6), +/// parameters_schema(7), source_url(8), trust_level(9), status(10), +/// created_at(11), updated_at(12) +#[cfg(feature = "libsql")] +fn libsql_row_to_tool_with_offset(row: &libsql::Row) -> Result { + libsql_row_to_tool_at(row, 0, 1, 2, 3, 4, 7, 8, 9, 10, 11, 12) +} + +#[cfg(feature = "libsql")] +#[allow(clippy::too_many_arguments)] +fn libsql_row_to_tool_at( + row: &libsql::Row, + id_idx: i32, + user_id_idx: i32, + name_idx: i32, + version_idx: i32, + description_idx: i32, + schema_idx: i32, + source_url_idx: i32, + trust_level_idx: i32, + status_idx: i32, + created_at_idx: i32, + updated_at_idx: i32, +) -> Result { + let id_str: String = row + .get(id_idx) + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + let trust_level_str: String = row + .get(trust_level_idx) + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + let status_str: String = row + .get(status_idx) + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + let schema_str: String = row + .get(schema_idx) + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + let created_at_str: String = row + .get(created_at_idx) + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + let updated_at_str: String = row + .get(updated_at_idx) + .map_err(|e| WasmStorageError::Database(e.to_string()))?; + + Ok(StoredWasmTool { + id: id_str + .parse() + .map_err(|e: uuid::Error| WasmStorageError::InvalidData(e.to_string()))?, + user_id: row + .get(user_id_idx) + .map_err(|e| WasmStorageError::Database(e.to_string()))?, + name: row + .get(name_idx) + .map_err(|e| WasmStorageError::Database(e.to_string()))?, + version: row + .get(version_idx) + .map_err(|e| WasmStorageError::Database(e.to_string()))?, + description: row + .get(description_idx) + .map_err(|e| WasmStorageError::Database(e.to_string()))?, + parameters_schema: serde_json::from_str(&schema_str).unwrap_or_default(), + source_url: row + .get::(source_url_idx) + .ok() + .filter(|s| !s.is_empty()), + trust_level: trust_level_str + .parse() + .map_err(WasmStorageError::InvalidData)?, + status: status_str.parse().map_err(WasmStorageError::InvalidData)?, + created_at: libsql_wasm_parse_ts(&created_at_str)?, + updated_at: libsql_wasm_parse_ts(&updated_at_str)?, + }) +} + #[cfg(test)] mod tests { use crate::tools::wasm::storage::{ diff --git a/src/workspace/mod.rs b/src/workspace/mod.rs index d0f05d9c..a6afb070 100644 --- a/src/workspace/mod.rs +++ b/src/workspace/mod.rs @@ -43,23 +43,206 @@ mod chunker; mod document; mod embeddings; +#[cfg(feature = "postgres")] mod repository; mod search; pub use chunker::{ChunkConfig, chunk_document}; pub use document::{MemoryChunk, MemoryDocument, WorkspaceEntry, paths}; pub use embeddings::{EmbeddingProvider, MockEmbeddings, NearAiEmbeddings, OpenAiEmbeddings}; +#[cfg(feature = "postgres")] pub use repository::Repository; -pub use search::{SearchConfig, SearchResult}; +pub use search::{RankedResult, SearchConfig, SearchResult, reciprocal_rank_fusion}; use std::sync::Arc; use chrono::{NaiveDate, Utc}; +#[cfg(feature = "postgres")] use deadpool_postgres::Pool; use uuid::Uuid; use crate::error::WorkspaceError; +/// Internal storage abstraction for Workspace. +/// +/// Allows Workspace to work with either a PostgreSQL `Repository` (the original +/// path) or any `Database` trait implementation (e.g. libSQL backend). +enum WorkspaceStorage { + /// PostgreSQL-backed repository (uses connection pool directly). + #[cfg(feature = "postgres")] + Repo(Repository), + /// Generic backend implementing the Database trait. + Db(Arc), +} + +impl WorkspaceStorage { + async fn get_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result { + match self { + #[cfg(feature = "postgres")] + Self::Repo(repo) => repo.get_document_by_path(user_id, agent_id, path).await, + Self::Db(db) => db.get_document_by_path(user_id, agent_id, path).await, + } + } + + async fn get_document_by_id(&self, id: Uuid) -> Result { + match self { + #[cfg(feature = "postgres")] + Self::Repo(repo) => repo.get_document_by_id(id).await, + Self::Db(db) => db.get_document_by_id(id).await, + } + } + + async fn get_or_create_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result { + match self { + #[cfg(feature = "postgres")] + Self::Repo(repo) => { + repo.get_or_create_document_by_path(user_id, agent_id, path) + .await + } + Self::Db(db) => { + db.get_or_create_document_by_path(user_id, agent_id, path) + .await + } + } + } + + async fn update_document(&self, id: Uuid, content: &str) -> Result<(), WorkspaceError> { + match self { + #[cfg(feature = "postgres")] + Self::Repo(repo) => repo.update_document(id, content).await, + Self::Db(db) => db.update_document(id, content).await, + } + } + + async fn delete_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result<(), WorkspaceError> { + match self { + #[cfg(feature = "postgres")] + Self::Repo(repo) => repo.delete_document_by_path(user_id, agent_id, path).await, + Self::Db(db) => db.delete_document_by_path(user_id, agent_id, path).await, + } + } + + async fn list_directory( + &self, + user_id: &str, + agent_id: Option, + directory: &str, + ) -> Result, WorkspaceError> { + match self { + #[cfg(feature = "postgres")] + Self::Repo(repo) => repo.list_directory(user_id, agent_id, directory).await, + Self::Db(db) => db.list_directory(user_id, agent_id, directory).await, + } + } + + async fn list_all_paths( + &self, + user_id: &str, + agent_id: Option, + ) -> Result, WorkspaceError> { + match self { + #[cfg(feature = "postgres")] + Self::Repo(repo) => repo.list_all_paths(user_id, agent_id).await, + Self::Db(db) => db.list_all_paths(user_id, agent_id).await, + } + } + + async fn delete_chunks(&self, document_id: Uuid) -> Result<(), WorkspaceError> { + match self { + #[cfg(feature = "postgres")] + Self::Repo(repo) => repo.delete_chunks(document_id).await, + Self::Db(db) => db.delete_chunks(document_id).await, + } + } + + async fn insert_chunk( + &self, + document_id: Uuid, + chunk_index: i32, + content: &str, + embedding: Option<&[f32]>, + ) -> Result { + match self { + #[cfg(feature = "postgres")] + Self::Repo(repo) => { + repo.insert_chunk(document_id, chunk_index, content, embedding) + .await + } + Self::Db(db) => { + db.insert_chunk(document_id, chunk_index, content, embedding) + .await + } + } + } + + async fn update_chunk_embedding( + &self, + chunk_id: Uuid, + embedding: &[f32], + ) -> Result<(), WorkspaceError> { + match self { + #[cfg(feature = "postgres")] + Self::Repo(repo) => repo.update_chunk_embedding(chunk_id, embedding).await, + Self::Db(db) => db.update_chunk_embedding(chunk_id, embedding).await, + } + } + + async fn get_chunks_without_embeddings( + &self, + user_id: &str, + agent_id: Option, + limit: usize, + ) -> Result, WorkspaceError> { + match self { + #[cfg(feature = "postgres")] + Self::Repo(repo) => { + repo.get_chunks_without_embeddings(user_id, agent_id, limit) + .await + } + Self::Db(db) => { + db.get_chunks_without_embeddings(user_id, agent_id, limit) + .await + } + } + } + + async fn hybrid_search( + &self, + user_id: &str, + agent_id: Option, + query: &str, + embedding: Option<&[f32]>, + config: &SearchConfig, + ) -> Result, WorkspaceError> { + match self { + #[cfg(feature = "postgres")] + Self::Repo(repo) => { + repo.hybrid_search(user_id, agent_id, query, embedding, config) + .await + } + Self::Db(db) => { + db.hybrid_search(user_id, agent_id, query, embedding, config) + .await + } + } + } +} + /// Default template seeded into HEARTBEAT.md on first access. /// /// Intentionally comment-only so the heartbeat runner treats it as @@ -80,25 +263,39 @@ const HEARTBEAT_SEED: &str = "\ /// Workspace provides database-backed memory storage for an agent. /// /// Each workspace is scoped to a user (and optionally an agent). -/// Documents are persisted to PostgreSQL and indexed for search. +/// Documents are persisted to the database and indexed for search. +/// Supports both PostgreSQL (via Repository) and libSQL (via Database trait). pub struct Workspace { /// User identifier (from channel). user_id: String, /// Optional agent ID for multi-agent isolation. agent_id: Option, - /// Database repository. - repo: Repository, + /// Database storage backend. + storage: WorkspaceStorage, /// Embedding provider for semantic search. embeddings: Option>, } impl Workspace { - /// Create a new workspace for a user. + /// Create a new workspace backed by a PostgreSQL connection pool. + #[cfg(feature = "postgres")] pub fn new(user_id: impl Into, pool: Pool) -> Self { Self { user_id: user_id.into(), agent_id: None, - repo: Repository::new(pool), + storage: WorkspaceStorage::Repo(Repository::new(pool)), + embeddings: None, + } + } + + /// Create a new workspace backed by any Database implementation. + /// + /// Use this for libSQL or any other backend that implements the Database trait. + pub fn new_with_db(user_id: impl Into, db: Arc) -> Self { + Self { + user_id: user_id.into(), + agent_id: None, + storage: WorkspaceStorage::Db(db), embeddings: None, } } @@ -138,7 +335,7 @@ impl Workspace { /// ``` pub async fn read(&self, path: &str) -> Result { let path = normalize_path(path); - self.repo + self.storage .get_document_by_path(&self.user_id, self.agent_id, &path) .await } @@ -155,14 +352,14 @@ impl Workspace { pub async fn write(&self, path: &str, content: &str) -> Result { let path = normalize_path(path); let doc = self - .repo + .storage .get_or_create_document_by_path(&self.user_id, self.agent_id, &path) .await?; - self.repo.update_document(doc.id, content).await?; + self.storage.update_document(doc.id, content).await?; self.reindex_document(doc.id).await?; // Return updated doc - self.repo.get_document_by_id(doc.id).await + self.storage.get_document_by_id(doc.id).await } /// Append content to a file. @@ -172,7 +369,7 @@ impl Workspace { pub async fn append(&self, path: &str, content: &str) -> Result<(), WorkspaceError> { let path = normalize_path(path); let doc = self - .repo + .storage .get_or_create_document_by_path(&self.user_id, self.agent_id, &path) .await?; @@ -182,7 +379,7 @@ impl Workspace { format!("{}\n{}", doc.content, content) }; - self.repo.update_document(doc.id, &new_content).await?; + self.storage.update_document(doc.id, &new_content).await?; self.reindex_document(doc.id).await?; Ok(()) } @@ -191,7 +388,7 @@ impl Workspace { pub async fn exists(&self, path: &str) -> Result { let path = normalize_path(path); match self - .repo + .storage .get_document_by_path(&self.user_id, self.agent_id, &path) .await { @@ -206,7 +403,7 @@ impl Workspace { /// Also deletes associated chunks. pub async fn delete(&self, path: &str) -> Result<(), WorkspaceError> { let path = normalize_path(path); - self.repo + self.storage .delete_document_by_path(&self.user_id, self.agent_id, &path) .await } @@ -229,14 +426,16 @@ impl Workspace { /// ``` pub async fn list(&self, directory: &str) -> Result, WorkspaceError> { let directory = normalize_directory(directory); - self.repo + self.storage .list_directory(&self.user_id, self.agent_id, &directory) .await } /// List all files recursively (flat list of all paths). pub async fn list_all(&self) -> Result, WorkspaceError> { - self.repo.list_all_paths(&self.user_id, self.agent_id).await + self.storage + .list_all_paths(&self.user_id, self.agent_id) + .await } // ==================== Convenience Methods ==================== @@ -280,7 +479,7 @@ impl Workspace { /// Helper to read or create a file. async fn read_or_create(&self, path: &str) -> Result { - self.repo + self.storage .get_or_create_document_by_path(&self.user_id, self.agent_id, path) .await } @@ -299,7 +498,7 @@ impl Workspace { } else { format!("{}\n\n{}", doc.content, entry) }; - self.repo.update_document(doc.id, &new_content).await?; + self.storage.update_document(doc.id, &new_content).await?; self.reindex_document(doc.id).await?; Ok(()) } @@ -395,7 +594,7 @@ impl Workspace { None }; - self.repo + self.storage .hybrid_search( &self.user_id, self.agent_id, @@ -411,13 +610,13 @@ impl Workspace { /// Re-index a document (chunk and generate embeddings). async fn reindex_document(&self, document_id: Uuid) -> Result<(), WorkspaceError> { // Get the document - let doc = self.repo.get_document_by_id(document_id).await?; + let doc = self.storage.get_document_by_id(document_id).await?; // Chunk the content let chunks = chunk_document(&doc.content, ChunkConfig::default()); // Delete old chunks - self.repo.delete_chunks(document_id).await?; + self.storage.delete_chunks(document_id).await?; // Insert new chunks for (index, content) in chunks.into_iter().enumerate() { @@ -434,7 +633,7 @@ impl Workspace { None }; - self.repo + self.storage .insert_chunk(document_id, index as i32, &content, embedding.as_deref()) .await?; } @@ -542,7 +741,7 @@ impl Workspace { }; let chunks = self - .repo + .storage .get_chunks_without_embeddings(&self.user_id, self.agent_id, 100) .await?; @@ -550,7 +749,7 @@ impl Workspace { for chunk in chunks { match provider.embed(&chunk.content).await { Ok(embedding) => { - self.repo + self.storage .update_chunk_embedding(chunk.id, &embedding) .await?; count += 1; diff --git a/tests/workspace_integration.rs b/tests/workspace_integration.rs index 11a4de76..dddd95e9 100644 --- a/tests/workspace_integration.rs +++ b/tests/workspace_integration.rs @@ -1,3 +1,4 @@ +#![cfg(feature = "postgres")] //! Integration tests for the workspace module. //! //! Requires a running PostgreSQL with pgvector extension.