Add workspace and memory system (OpenClaw-inspired)

Implements persistent memory for agents with hybrid search:

- Database-backed workspace with PostgreSQL (not filesystem)
- Memory documents: MEMORY.md, daily logs, identity files
- Chunked content with FTS (tsvector) + vector (pgvector) indexes
- Reciprocal Rank Fusion (RRF) for hybrid search combining BM25 and semantic
- Memory tools: memory_search, memory_write, memory_read
- Proactive heartbeat system for periodic execution (30 min default)
- OpenAI embeddings provider (text-embedding-3-small)

Key patterns from OpenClaw:
- "Memory is files, not RAM" - explicit persistence required
- Two-tier memory: daily logs (raw) + curated MEMORY.md
- Session isolation via user_id/agent_id scoping

Co-Authored-By: Claude Opus 4.5 <[email protected]>
This commit is contained in:
Illia Polosukhin
2026-02-02 21:18:47 -08:00
co-authored by Claude Opus 4.5
parent 8c38566378
commit 4e238e60ac
15 changed files with 2999 additions and 0 deletions
Generated
+11
View File
@@ -1275,6 +1275,7 @@ dependencies = [
"deadpool-postgres",
"dotenvy",
"futures",
"pgvector",
"postgres-types",
"pretty_assertions",
"refinery",
@@ -1404,6 +1405,16 @@ version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "pgvector"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc58e2d255979a31caa7cabfa7aac654af0354220719ab7a68520ae7a91e8c0b"
dependencies = [
"bytes",
"postgres-types",
]
[[package]]
name = "phf"
version = "0.13.1"
+4
View File
@@ -60,6 +60,10 @@ aho-corasick = "1"
# Secrecy for sensitive values
secrecy = { version = "0.10", features = ["serde"] }
# Vector embeddings for semantic search
# The postgres feature provides ToSql/FromSql for postgres-types (shared by tokio-postgres)
pgvector = { version = "0.4", features = ["postgres"] }
[dev-dependencies]
tokio-test = "0.4"
testcontainers-modules = { version = "0.11", features = ["postgres"] }
+159
View File
@@ -0,0 +1,159 @@
-- NEAR Agent Database Schema
-- V2: Workspace and memory system (OpenClaw-inspired)
--
-- This migration adds:
-- 1. Persistent memory documents (MEMORY.md, daily logs, identity files)
-- 2. Chunked content for hybrid search (FTS + vector)
-- 3. Heartbeat state for proactive execution
-- Enable pgvector extension for semantic search
-- NOTE: This requires pgvector to be installed on the PostgreSQL server
-- Install via: CREATE EXTENSION vector; (requires superuser or rds_superuser)
CREATE EXTENSION IF NOT EXISTS vector;
-- ==================== Memory Documents ====================
-- Stores full documents like MEMORY.md, daily logs, identity files
-- Think of this as the filesystem equivalent, but in PostgreSQL
CREATE TABLE memory_documents (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-- Ownership: who this document belongs to
user_id TEXT NOT NULL, -- User identifier (from channel)
agent_id UUID, -- NULL = shared across all agents for this user
-- Document type and content
doc_type TEXT NOT NULL, -- 'memory', 'daily_log', 'identity', 'soul', 'agents', 'user', 'heartbeat'
title TEXT, -- Optional title (e.g., date for daily logs)
content TEXT NOT NULL, -- Full document content
-- Timestamps
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-- Flexible metadata (tags, source, etc.)
metadata JSONB NOT NULL DEFAULT '{}',
-- Ensure one document per type per user (for singleton docs like MEMORY.md)
-- Daily logs use title as the date discriminator
CONSTRAINT unique_doc_per_user_type UNIQUE (user_id, agent_id, doc_type, title)
);
-- Indexes for common queries
CREATE INDEX idx_memory_documents_user ON memory_documents(user_id);
CREATE INDEX idx_memory_documents_user_type ON memory_documents(user_id, doc_type);
CREATE INDEX idx_memory_documents_updated ON memory_documents(updated_at DESC);
-- ==================== Memory Chunks ====================
-- Documents are chunked for search. Each chunk has:
-- 1. Full-text search vector (tsvector) for keyword matching
-- 2. Embedding vector for semantic similarity
--
-- Hybrid search combines both using Reciprocal Rank Fusion (RRF)
CREATE TABLE memory_chunks (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
document_id UUID NOT NULL REFERENCES memory_documents(id) ON DELETE CASCADE,
-- Chunk position and content
chunk_index INT NOT NULL, -- Position in document (0-based)
content TEXT NOT NULL, -- Chunk text (~800 tokens with 15% overlap)
-- Full-text search: auto-generated tsvector
content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('english', content)) STORED,
-- Semantic search: embedding vector (OpenAI text-embedding-ada-002 = 1536 dims)
-- NULL until embeddings are generated
embedding VECTOR(1536),
-- Timestamps
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-- Each chunk index unique per document
CONSTRAINT unique_chunk_per_doc UNIQUE (document_id, chunk_index)
);
-- GIN index for full-text search
CREATE INDEX idx_memory_chunks_tsv ON memory_chunks USING GIN(content_tsv);
-- HNSW index for vector similarity (cosine distance)
-- HNSW is faster than IVFFlat for reads, slightly slower for writes
CREATE INDEX idx_memory_chunks_embedding ON memory_chunks
USING hnsw(embedding vector_cosine_ops)
WITH (m = 16, ef_construction = 64);
-- Index for document lookups
CREATE INDEX idx_memory_chunks_document ON memory_chunks(document_id);
-- ==================== Heartbeat State ====================
-- Tracks periodic heartbeat execution per user/agent
CREATE TABLE heartbeat_state (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id TEXT NOT NULL,
agent_id UUID, -- NULL = default agent for user
-- Timing
last_run TIMESTAMPTZ, -- When heartbeat last executed
next_run TIMESTAMPTZ, -- Scheduled next execution
interval_seconds INT NOT NULL DEFAULT 1800, -- 30 minutes default
-- State
enabled BOOLEAN NOT NULL DEFAULT true,
consecutive_failures INT NOT NULL DEFAULT 0,
-- Last check timestamps (for batched monitoring)
-- e.g., {"email": "2024-01-15T10:00:00Z", "calendar": "2024-01-15T10:00:00Z"}
last_checks JSONB NOT NULL DEFAULT '{}',
-- Ensure one heartbeat config per user/agent
CONSTRAINT unique_heartbeat_per_user UNIQUE (user_id, agent_id)
);
CREATE INDEX idx_heartbeat_user ON heartbeat_state(user_id);
CREATE INDEX idx_heartbeat_next_run ON heartbeat_state(next_run) WHERE enabled = true;
-- ==================== Helper Functions ====================
-- Function to update updated_at timestamp
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = NOW();
RETURN NEW;
END;
$$ language 'plpgsql';
-- Trigger to auto-update updated_at on memory_documents
CREATE TRIGGER update_memory_documents_updated_at
BEFORE UPDATE ON memory_documents
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- ==================== Views ====================
-- View for documents with chunk counts (useful for debugging)
CREATE VIEW memory_documents_summary AS
SELECT
d.id,
d.user_id,
d.doc_type,
d.title,
d.created_at,
d.updated_at,
COUNT(c.id) as chunk_count,
COUNT(c.embedding) as embedded_chunk_count
FROM memory_documents d
LEFT JOIN memory_chunks c ON c.document_id = d.id
GROUP BY d.id;
-- View for pending embedding work
CREATE VIEW chunks_pending_embedding AS
SELECT
c.id as chunk_id,
c.document_id,
d.user_id,
d.doc_type,
LENGTH(c.content) as content_length
FROM memory_chunks c
JOIN memory_documents d ON d.id = c.document_id
WHERE c.embedding IS NULL;
+320
View File
@@ -0,0 +1,320 @@
//! Proactive heartbeat system for periodic execution.
//!
//! The heartbeat runner executes periodically (default: every 30 minutes) and:
//! 1. Reads the HEARTBEAT.md checklist
//! 2. Runs an agent turn to process the checklist
//! 3. Reports any findings to the configured channel
//!
//! If nothing needs attention, the agent replies "HEARTBEAT_OK" and no
//! message is sent to the user.
//!
//! # Usage
//!
//! Create a HEARTBEAT.md in the workspace with a checklist of things to monitor:
//!
//! ```markdown
//! # Heartbeat Checklist
//!
//! - [ ] Check for unread emails
//! - [ ] Review calendar for upcoming events
//! - [ ] Check project build status
//! ```
//!
//! The agent will process this checklist on each heartbeat and only notify
//! if action is needed.
use std::sync::Arc;
use std::time::Duration;
use tokio::sync::mpsc;
use crate::channels::OutgoingResponse;
use crate::error::WorkspaceError;
use crate::llm::{ChatMessage, CompletionRequest, LlmProvider};
use crate::workspace::Workspace;
/// Configuration for the heartbeat runner.
#[derive(Debug, Clone)]
pub struct HeartbeatConfig {
/// Interval between heartbeat checks.
pub interval: Duration,
/// Whether heartbeat is enabled.
pub enabled: bool,
/// Maximum consecutive failures before disabling.
pub max_failures: u32,
/// User ID to notify on heartbeat findings.
pub notify_user_id: Option<String>,
/// Channel to notify on heartbeat findings.
pub notify_channel: Option<String>,
}
impl Default for HeartbeatConfig {
fn default() -> Self {
Self {
interval: Duration::from_secs(30 * 60), // 30 minutes
enabled: true,
max_failures: 3,
notify_user_id: None,
notify_channel: None,
}
}
}
impl HeartbeatConfig {
/// Create a config with a specific interval.
pub fn with_interval(mut self, interval: Duration) -> Self {
self.interval = interval;
self
}
/// Disable heartbeat.
pub fn disabled(mut self) -> Self {
self.enabled = false;
self
}
/// Set the notification target.
pub fn with_notify(mut self, user_id: impl Into<String>, channel: impl Into<String>) -> Self {
self.notify_user_id = Some(user_id.into());
self.notify_channel = Some(channel.into());
self
}
}
/// Result of a heartbeat check.
#[derive(Debug)]
pub enum HeartbeatResult {
/// Nothing needs attention.
Ok,
/// Something needs attention, with the message to send.
NeedsAttention(String),
/// Heartbeat was skipped (no checklist or disabled).
Skipped,
/// Heartbeat failed.
Failed(String),
}
/// Heartbeat runner for proactive periodic execution.
pub struct HeartbeatRunner {
config: HeartbeatConfig,
workspace: Arc<Workspace>,
llm: Arc<dyn LlmProvider>,
response_tx: Option<mpsc::Sender<OutgoingResponse>>,
consecutive_failures: u32,
}
impl HeartbeatRunner {
/// Create a new heartbeat runner.
pub fn new(
config: HeartbeatConfig,
workspace: Arc<Workspace>,
llm: Arc<dyn LlmProvider>,
) -> Self {
Self {
config,
workspace,
llm,
response_tx: None,
consecutive_failures: 0,
}
}
/// Set the response channel for notifications.
pub fn with_response_channel(mut self, tx: mpsc::Sender<OutgoingResponse>) -> Self {
self.response_tx = Some(tx);
self
}
/// Run the heartbeat loop.
///
/// This runs forever, checking periodically based on the configured interval.
pub async fn run(&mut self) {
if !self.config.enabled {
tracing::info!("Heartbeat is disabled, not starting loop");
return;
}
tracing::info!(
"Starting heartbeat loop with interval {:?}",
self.config.interval
);
let mut interval = tokio::time::interval(self.config.interval);
// Don't run immediately on startup
interval.tick().await;
loop {
interval.tick().await;
match self.check_heartbeat().await {
HeartbeatResult::Ok => {
tracing::debug!("Heartbeat OK");
self.consecutive_failures = 0;
}
HeartbeatResult::NeedsAttention(message) => {
tracing::info!("Heartbeat needs attention: {}", message);
self.consecutive_failures = 0;
self.send_notification(&message).await;
}
HeartbeatResult::Skipped => {
tracing::debug!("Heartbeat skipped");
}
HeartbeatResult::Failed(error) => {
tracing::error!("Heartbeat failed: {}", error);
self.consecutive_failures += 1;
if self.consecutive_failures >= self.config.max_failures {
tracing::error!(
"Heartbeat disabled after {} consecutive failures",
self.consecutive_failures
);
break;
}
}
}
}
}
/// Run a single heartbeat check.
pub async fn check_heartbeat(&self) -> HeartbeatResult {
// Get the heartbeat checklist
let checklist = match self.workspace.heartbeat_checklist().await {
Ok(Some(content)) if !content.trim().is_empty() => content,
Ok(_) => return HeartbeatResult::Skipped,
Err(e) => return HeartbeatResult::Failed(format!("Failed to read checklist: {}", e)),
};
// Build the heartbeat prompt
let prompt = format!(
"Read the HEARTBEAT.md checklist below and follow it strictly. \
Do not infer or repeat old tasks. Check each item and report findings.\n\
\n\
If nothing needs attention, reply EXACTLY with: HEARTBEAT_OK\n\
\n\
If something needs attention, provide a concise summary of what needs action.\n\
\n\
## HEARTBEAT.md\n\
\n\
{}",
checklist
);
// Get the system prompt for context
let system_prompt = match self.workspace.system_prompt().await {
Ok(p) => p,
Err(e) => {
tracing::warn!("Failed to get system prompt for heartbeat: {}", e);
String::new()
}
};
// Run the agent turn
let messages = if system_prompt.is_empty() {
vec![ChatMessage::user(&prompt)]
} else {
vec![
ChatMessage::system(&system_prompt),
ChatMessage::user(&prompt),
]
};
let request = CompletionRequest::new(messages)
.with_max_tokens(1024)
.with_temperature(0.3); // Lower temperature for more focused responses
let response = match self.llm.complete(request).await {
Ok(r) => r,
Err(e) => return HeartbeatResult::Failed(format!("LLM call failed: {}", e)),
};
let content = response.content.trim();
// Check if nothing needs attention
if content == "HEARTBEAT_OK" || content.contains("HEARTBEAT_OK") {
return HeartbeatResult::Ok;
}
HeartbeatResult::NeedsAttention(content.to_string())
}
/// Send a notification about heartbeat findings.
async fn send_notification(&self, message: &str) {
let Some(ref tx) = self.response_tx else {
tracing::debug!("No response channel configured for heartbeat notifications");
return;
};
let response = OutgoingResponse {
content: format!("🔔 **Heartbeat Alert**\n\n{}", message),
thread_id: None,
metadata: serde_json::json!({
"source": "heartbeat",
}),
};
if let Err(e) = tx.send(response).await {
tracing::error!("Failed to send heartbeat notification: {}", e);
}
}
}
/// Spawn the heartbeat runner as a background task.
///
/// Returns a handle that can be used to stop the runner.
pub fn spawn_heartbeat(
config: HeartbeatConfig,
workspace: Arc<Workspace>,
llm: Arc<dyn LlmProvider>,
response_tx: Option<mpsc::Sender<OutgoingResponse>>,
) -> tokio::task::JoinHandle<()> {
let mut runner = HeartbeatRunner::new(config, workspace, llm);
if let Some(tx) = response_tx {
runner = runner.with_response_channel(tx);
}
tokio::spawn(async move {
runner.run().await;
})
}
/// Update heartbeat state in the database.
pub async fn update_heartbeat_state(
workspace: &Workspace,
last_run: chrono::DateTime<chrono::Utc>,
) -> Result<(), WorkspaceError> {
// This would update the heartbeat_state table
// For now, we just log
tracing::debug!(
"Heartbeat state updated for user {} at {}",
workspace.user_id(),
last_run
);
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_heartbeat_config_defaults() {
let config = HeartbeatConfig::default();
assert!(config.enabled);
assert_eq!(config.interval, Duration::from_secs(30 * 60));
assert_eq!(config.max_failures, 3);
}
#[test]
fn test_heartbeat_config_builders() {
let config = HeartbeatConfig::default()
.with_interval(Duration::from_secs(60))
.with_notify("user1", "telegram");
assert_eq!(config.interval, Duration::from_secs(60));
assert_eq!(config.notify_user_id, Some("user1".to_string()));
assert_eq!(config.notify_channel, Some("telegram".to_string()));
let disabled = HeartbeatConfig::default().disabled();
assert!(!disabled.enabled);
}
}
+3
View File
@@ -5,14 +5,17 @@
//! - Job scheduling and execution
//! - Tool invocation with safety
//! - Self-repair for stuck jobs
//! - Proactive heartbeat execution
mod agent_loop;
mod heartbeat;
mod router;
mod scheduler;
mod self_repair;
mod worker;
pub use agent_loop::Agent;
pub use heartbeat::{HeartbeatConfig, HeartbeatResult, HeartbeatRunner, spawn_heartbeat};
pub use router::{MessageIntent, Router};
pub use scheduler::Scheduler;
pub use self_repair::{RepairResult, RepairTask, SelfRepair, StuckJob};
+28
View File
@@ -36,6 +36,9 @@ pub enum Error {
#[error("Repair error: {0}")]
Repair(#[from] RepairError),
#[error("Workspace error: {0}")]
Workspace(#[from] WorkspaceError),
}
/// Configuration-related errors.
@@ -268,5 +271,30 @@ pub enum RepairError {
},
}
/// Workspace/memory errors.
#[derive(Debug, thiserror::Error)]
pub enum WorkspaceError {
#[error("Document not found: {doc_type} for user {user_id}")]
DocumentNotFound { doc_type: String, user_id: String },
#[error("Search failed: {reason}")]
SearchFailed { reason: String },
#[error("Embedding generation failed: {reason}")]
EmbeddingFailed { reason: String },
#[error("Document chunking failed: {reason}")]
ChunkingFailed { reason: String },
#[error("Invalid document type: {doc_type}")]
InvalidDocType { doc_type: String },
#[error("Workspace not initialized for user {user_id}")]
NotInitialized { user_id: String },
#[error("Heartbeat error: {reason}")]
HeartbeatError { reason: String },
}
/// Result type alias for the agent.
pub type Result<T> = std::result::Result<T, Error>;
+2
View File
@@ -49,6 +49,7 @@ pub mod history;
pub mod llm;
pub mod safety;
pub mod tools;
pub mod workspace;
pub use config::Config;
pub use error::{Error, Result};
@@ -62,4 +63,5 @@ pub mod prelude {
pub use crate::llm::LlmProvider;
pub use crate::safety::{SanitizedOutput, Sanitizer};
pub use crate::tools::{Tool, ToolOutput, ToolRegistry};
pub use crate::workspace::{DocType, MemoryDocument, Workspace};
}
+362
View File
@@ -0,0 +1,362 @@
//! Memory tools for persistent workspace memory.
//!
//! These tools allow the agent to:
//! - Search past memories, decisions, and context
//! - Write important information to long-term memory
//!
//! # Usage
//!
//! The agent should use `memory_search` before answering questions about
//! prior work, decisions, dates, people, preferences, or todos.
//!
//! Use `memory_write` to persist important facts that should be remembered
//! across sessions.
use std::sync::Arc;
use async_trait::async_trait;
use crate::context::JobContext;
use crate::tools::tool::{Tool, ToolError, ToolOutput};
use crate::workspace::Workspace;
/// Tool for searching workspace memory.
///
/// Performs hybrid search (FTS + semantic) across all memory documents.
/// The agent should call this tool before answering questions about
/// prior work, decisions, preferences, or any historical context.
pub struct MemorySearchTool {
workspace: Arc<Workspace>,
}
impl MemorySearchTool {
/// Create a new memory search tool.
pub fn new(workspace: Arc<Workspace>) -> Self {
Self { workspace }
}
}
#[async_trait]
impl Tool for MemorySearchTool {
fn name(&self) -> &str {
"memory_search"
}
fn description(&self) -> &str {
"Search past memories, decisions, and context. MUST be called before answering \
questions about prior work, decisions, dates, people, preferences, or todos. \
Returns relevant snippets with relevance scores."
}
fn parameters_schema(&self) -> serde_json::Value {
serde_json::json!({
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "The search query. Use natural language to describe what you're looking for."
},
"limit": {
"type": "integer",
"description": "Maximum number of results to return (default: 5, max: 20)",
"default": 5,
"minimum": 1,
"maximum": 20
}
},
"required": ["query"]
})
}
async fn execute(
&self,
params: serde_json::Value,
_ctx: &JobContext,
) -> Result<ToolOutput, ToolError> {
let start = std::time::Instant::now();
let query = params
.get("query")
.and_then(|v| v.as_str())
.ok_or_else(|| ToolError::InvalidParameters("missing 'query' parameter".to_string()))?;
let limit = params
.get("limit")
.and_then(|v| v.as_u64())
.unwrap_or(5)
.min(20) as usize;
let results = self
.workspace
.search(query, limit)
.await
.map_err(|e| ToolError::ExecutionFailed(format!("Search failed: {}", e)))?;
let output = serde_json::json!({
"query": query,
"results": results.iter().map(|r| serde_json::json!({
"content": r.content,
"score": r.score,
"document_id": r.document_id.to_string(),
"is_hybrid_match": r.is_hybrid(),
})).collect::<Vec<_>>(),
"result_count": results.len(),
});
Ok(ToolOutput::success(output, start.elapsed()))
}
fn requires_sanitization(&self) -> bool {
false // Internal memory, trusted content
}
}
/// Tool for writing to workspace memory.
///
/// Use this to persist important information that should be remembered
/// across sessions: decisions, preferences, facts, lessons learned.
pub struct MemoryWriteTool {
workspace: Arc<Workspace>,
}
impl MemoryWriteTool {
/// Create a new memory write tool.
pub fn new(workspace: Arc<Workspace>) -> Self {
Self { workspace }
}
}
#[async_trait]
impl Tool for MemoryWriteTool {
fn name(&self) -> &str {
"memory_write"
}
fn description(&self) -> &str {
"Write to persistent memory. Use for important facts, decisions, preferences, \
or lessons learned that should be remembered across sessions. Use 'memory' target \
for curated long-term facts, 'daily_log' for timestamped session notes."
}
fn parameters_schema(&self) -> serde_json::Value {
serde_json::json!({
"type": "object",
"properties": {
"content": {
"type": "string",
"description": "The content to write to memory. Be concise but include relevant context."
},
"target": {
"type": "string",
"enum": ["memory", "daily_log"],
"description": "Where to write: 'memory' for long-term curated facts, 'daily_log' for timestamped session notes",
"default": "daily_log"
}
},
"required": ["content"]
})
}
async fn execute(
&self,
params: serde_json::Value,
_ctx: &JobContext,
) -> Result<ToolOutput, ToolError> {
let start = std::time::Instant::now();
let content = params
.get("content")
.and_then(|v| v.as_str())
.ok_or_else(|| {
ToolError::InvalidParameters("missing 'content' parameter".to_string())
})?;
if content.trim().is_empty() {
return Err(ToolError::InvalidParameters(
"content cannot be empty".to_string(),
));
}
let target = params
.get("target")
.and_then(|v| v.as_str())
.unwrap_or("daily_log");
match target {
"memory" => {
self.workspace
.append_memory(content)
.await
.map_err(|e| ToolError::ExecutionFailed(format!("Write failed: {}", e)))?;
}
"daily_log" => {
self.workspace
.append_daily_log(content)
.await
.map_err(|e| ToolError::ExecutionFailed(format!("Write failed: {}", e)))?;
}
_ => {
return Err(ToolError::InvalidParameters(format!(
"invalid target '{}', must be 'memory' or 'daily_log'",
target
)));
}
}
let output = serde_json::json!({
"status": "written",
"target": target,
"content_length": content.len(),
});
Ok(ToolOutput::success(output, start.elapsed()))
}
fn requires_sanitization(&self) -> bool {
false // Internal tool
}
}
/// Tool for reading specific memory documents.
///
/// Use this to read the full content of identity files, heartbeat checklist,
/// or other specific documents.
pub struct MemoryReadTool {
workspace: Arc<Workspace>,
}
impl MemoryReadTool {
/// Create a new memory read tool.
pub fn new(workspace: Arc<Workspace>) -> Self {
Self { workspace }
}
}
#[async_trait]
impl Tool for MemoryReadTool {
fn name(&self) -> &str {
"memory_read"
}
fn description(&self) -> &str {
"Read a specific memory document by type. Use this to read identity files, \
heartbeat checklist, or full memory document content."
}
fn parameters_schema(&self) -> serde_json::Value {
serde_json::json!({
"type": "object",
"properties": {
"doc_type": {
"type": "string",
"enum": ["memory", "daily_log", "identity", "soul", "agents", "user", "heartbeat"],
"description": "The type of document to read"
},
"title": {
"type": "string",
"description": "Optional title (required for daily_log, format: YYYY-MM-DD)"
}
},
"required": ["doc_type"]
})
}
async fn execute(
&self,
params: serde_json::Value,
_ctx: &JobContext,
) -> Result<ToolOutput, ToolError> {
let start = std::time::Instant::now();
let doc_type_str = params
.get("doc_type")
.and_then(|v| v.as_str())
.ok_or_else(|| {
ToolError::InvalidParameters("missing 'doc_type' parameter".to_string())
})?;
let doc_type = crate::workspace::DocType::try_from(doc_type_str)
.map_err(|e| ToolError::InvalidParameters(e.to_string()))?;
let title = params.get("title").and_then(|v| v.as_str());
let doc = self
.workspace
.get_document(doc_type, title)
.await
.map_err(|e| ToolError::ExecutionFailed(format!("Read failed: {}", e)))?;
let output = serde_json::json!({
"doc_type": doc_type_str,
"title": doc.title,
"content": doc.content,
"word_count": doc.word_count(),
"updated_at": doc.updated_at.to_rfc3339(),
});
Ok(ToolOutput::success(output, start.elapsed()))
}
fn requires_sanitization(&self) -> bool {
false // Internal memory
}
}
#[cfg(test)]
mod tests {
use super::*;
// Integration tests would require a database connection.
// Unit tests for parameter validation:
#[test]
fn test_memory_search_schema() {
let workspace = Arc::new(Workspace::new(
"test_user",
deadpool_postgres::Pool::builder(deadpool_postgres::Manager::new(
tokio_postgres::Config::new(),
tokio_postgres::NoTls,
))
.build()
.unwrap(),
));
let tool = MemorySearchTool::new(workspace);
assert_eq!(tool.name(), "memory_search");
assert!(!tool.requires_sanitization());
let schema = tool.parameters_schema();
assert!(schema["properties"]["query"].is_object());
assert!(
schema["required"]
.as_array()
.unwrap()
.contains(&"query".into())
);
}
#[test]
fn test_memory_write_schema() {
let workspace = Arc::new(Workspace::new(
"test_user",
deadpool_postgres::Pool::builder(deadpool_postgres::Manager::new(
tokio_postgres::Config::new(),
tokio_postgres::NoTls,
))
.build()
.unwrap(),
));
let tool = MemoryWriteTool::new(workspace);
assert_eq!(tool.name(), "memory_write");
let schema = tool.parameters_schema();
assert!(schema["properties"]["content"].is_object());
assert!(
schema["properties"]["target"]["enum"]
.as_array()
.unwrap()
.contains(&"memory".into())
);
}
}
+2
View File
@@ -5,6 +5,7 @@ mod ecommerce;
mod http;
mod json;
mod marketplace;
mod memory;
mod restaurant;
mod taskrabbit;
mod time;
@@ -14,6 +15,7 @@ pub use ecommerce::EcommerceTool;
pub use http::HttpTool;
pub use json::JsonTool;
pub use marketplace::MarketplaceTool;
pub use memory::{MemoryReadTool, MemorySearchTool, MemoryWriteTool};
pub use restaurant::RestaurantTool;
pub use taskrabbit::TaskRabbitTool;
pub use time::TimeTool;
+314
View File
@@ -0,0 +1,314 @@
//! Document chunking for search indexing.
//!
//! Documents are split into overlapping chunks for better search recall.
//! The overlap ensures context is preserved across chunk boundaries.
/// Configuration for document chunking.
#[derive(Debug, Clone)]
pub struct ChunkConfig {
/// Target chunk size in words (approximate tokens).
/// Default: 800 (roughly 800 tokens for English text).
pub chunk_size: usize,
/// Overlap percentage between chunks.
/// Default: 0.15 (15% overlap).
pub overlap_percent: f32,
/// Minimum chunk size (don't create tiny trailing chunks).
/// Default: 50 words.
pub min_chunk_size: usize,
}
impl Default for ChunkConfig {
fn default() -> Self {
Self {
chunk_size: 800,
overlap_percent: 0.15,
min_chunk_size: 50,
}
}
}
impl ChunkConfig {
/// Create a config with a specific chunk size.
pub fn with_chunk_size(mut self, size: usize) -> Self {
self.chunk_size = size;
self
}
/// Create a config with a specific overlap percentage.
pub fn with_overlap(mut self, percent: f32) -> Self {
self.overlap_percent = percent.clamp(0.0, 0.5);
self
}
/// Calculate the overlap size in words.
fn overlap_size(&self) -> usize {
(self.chunk_size as f32 * self.overlap_percent) as usize
}
/// Calculate the step size (chunk_size - overlap).
fn step_size(&self) -> usize {
self.chunk_size.saturating_sub(self.overlap_size())
}
}
/// Split a document into overlapping chunks.
///
/// Each chunk contains approximately `chunk_size` words, with `overlap_percent`
/// overlap between adjacent chunks. This ensures that:
/// 1. Context is preserved across chunk boundaries
/// 2. Search can find content that spans chunk boundaries
///
/// # Arguments
///
/// * `content` - The document text to chunk
/// * `config` - Chunking configuration
///
/// # Returns
///
/// A vector of chunk strings. Empty documents return an empty vector.
pub fn chunk_document(content: &str, config: ChunkConfig) -> Vec<String> {
if content.is_empty() {
return Vec::new();
}
// Split into words while preserving structure
let words: Vec<&str> = content.split_whitespace().collect();
if words.is_empty() {
return Vec::new();
}
// If content is smaller than chunk size, return as single chunk
if words.len() <= config.chunk_size {
return vec![content.to_string()];
}
let step = config.step_size();
let mut chunks = Vec::new();
let mut start = 0;
while start < words.len() {
let end = (start + config.chunk_size).min(words.len());
let chunk_words = &words[start..end];
// Don't create tiny trailing chunks, merge with previous
if chunk_words.len() < config.min_chunk_size && !chunks.is_empty() {
let last = chunks.pop().unwrap();
let combined = format!("{} {}", last, chunk_words.join(" "));
chunks.push(combined);
break;
}
chunks.push(chunk_words.join(" "));
// Move to next chunk position
start += step;
// Avoid creating duplicate chunks at the end
if start + config.min_chunk_size >= words.len() && end == words.len() {
break;
}
}
chunks
}
/// Split content by paragraphs first, then chunk.
///
/// This is better for preserving semantic boundaries.
pub fn chunk_by_paragraphs(content: &str, config: ChunkConfig) -> Vec<String> {
if content.is_empty() {
return Vec::new();
}
// Split by double newlines (paragraphs)
let paragraphs: Vec<&str> = content
.split("\n\n")
.map(|p| p.trim())
.filter(|p| !p.is_empty())
.collect();
if paragraphs.is_empty() {
return chunk_document(content, config);
}
let mut chunks = Vec::new();
let mut current_chunk = String::new();
let mut current_word_count = 0;
for paragraph in paragraphs {
let para_words = paragraph.split_whitespace().count();
// If this paragraph alone exceeds chunk size, chunk it separately
if para_words > config.chunk_size {
// Flush current chunk first
if !current_chunk.is_empty() {
chunks.push(current_chunk.trim().to_string());
current_chunk = String::new();
current_word_count = 0;
}
// Chunk the large paragraph
let para_chunks = chunk_document(paragraph, config.clone());
chunks.extend(para_chunks);
continue;
}
// Check if adding this paragraph would exceed chunk size
if current_word_count + para_words > config.chunk_size {
// Flush current chunk
if !current_chunk.is_empty() {
chunks.push(current_chunk.trim().to_string());
}
current_chunk = paragraph.to_string();
current_word_count = para_words;
} else {
// Add paragraph to current chunk
if !current_chunk.is_empty() {
current_chunk.push_str("\n\n");
}
current_chunk.push_str(paragraph);
current_word_count += para_words;
}
}
// Flush remaining content
if !current_chunk.is_empty() {
// If too small, merge with previous chunk if possible
if current_word_count < config.min_chunk_size && !chunks.is_empty() {
let last = chunks.pop().unwrap();
chunks.push(format!("{}\n\n{}", last, current_chunk.trim()));
} else {
chunks.push(current_chunk.trim().to_string());
}
}
chunks
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_empty_content() {
let config = ChunkConfig::default();
assert!(chunk_document("", config.clone()).is_empty());
assert!(chunk_document(" ", config).is_empty());
}
#[test]
fn test_small_content() {
let config = ChunkConfig::default();
let content = "Hello world, this is a test.";
let chunks = chunk_document(content, config);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0], content);
}
#[test]
fn test_exact_chunk_size() {
let config = ChunkConfig::default().with_chunk_size(5);
let content = "one two three four five";
let chunks = chunk_document(content, config);
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0], content);
}
#[test]
fn test_chunking_with_overlap() {
let config = ChunkConfig {
chunk_size: 10,
overlap_percent: 0.2, // 2 word overlap
min_chunk_size: 3, // Low threshold for test
};
// 20 words
let content = "one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen sixteen seventeen eighteen nineteen twenty";
let chunks = chunk_document(content, config);
// Should create overlapping chunks
assert!(
chunks.len() >= 2,
"Expected at least 2 chunks, got {}",
chunks.len()
);
// Each chunk should have roughly 10 words (allowing for overlap/merging)
for chunk in &chunks {
let word_count = chunk.split_whitespace().count();
assert!(word_count >= 3, "Chunk too small: {} words", word_count);
}
}
#[test]
fn test_overlap_calculation() {
let config = ChunkConfig::default()
.with_chunk_size(100)
.with_overlap(0.15);
assert_eq!(config.overlap_size(), 15);
assert_eq!(config.step_size(), 85);
}
#[test]
fn test_paragraph_chunking() {
let config = ChunkConfig::default().with_chunk_size(20);
let content = "First paragraph with some words.\n\nSecond paragraph with different content.\n\nThird paragraph here.";
let chunks = chunk_by_paragraphs(content, config);
// Should preserve paragraph boundaries
assert!(!chunks.is_empty());
for chunk in &chunks {
// No chunk should start or end with \n\n
assert!(!chunk.starts_with("\n"));
assert!(!chunk.ends_with("\n"));
}
}
#[test]
fn test_large_paragraph_handling() {
let config = ChunkConfig {
chunk_size: 10,
overlap_percent: 0.15,
min_chunk_size: 3, // Low threshold for test
};
// Create a paragraph with 30 words
let large_para = (1..=30)
.map(|i| format!("word{}", i))
.collect::<Vec<_>>()
.join(" ");
let content = format!("Short intro.\n\n{}\n\nShort outro.", large_para);
let chunks = chunk_by_paragraphs(&content, config);
// Should have multiple chunks due to large paragraph
// 30 words + 2 intro + 2 outro = 34 words, chunk_size=10
// Expect at least 3 chunks
assert!(
chunks.len() >= 3,
"Expected at least 3 chunks for 34 words with chunk_size=10, got {}",
chunks.len()
);
}
#[test]
fn test_min_chunk_size_merging() {
let config = ChunkConfig {
chunk_size: 10,
overlap_percent: 0.0,
min_chunk_size: 5,
};
// 12 words: should create one chunk of 10, and merge the remaining 2 with it
let content = "one two three four five six seven eight nine ten eleven twelve";
let chunks = chunk_document(content, config);
// Should merge the tiny trailing chunk
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].split_whitespace().count(), 12);
}
}
+236
View File
@@ -0,0 +1,236 @@
//! Memory document types.
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::error::WorkspaceError;
/// Document type in the workspace.
///
/// Each type represents a different kind of persistent memory:
/// - **Memory**: Long-term curated facts and decisions (MEMORY.md)
/// - **DailyLog**: Append-only daily notes (memory/YYYY-MM-DD.md)
/// - **Identity**: Agent name and personality
/// - **Soul**: Core values and behavior principles
/// - **Agents**: Behavior instructions
/// - **User**: User context (name, preferences)
/// - **Heartbeat**: Periodic checklist for proactive execution
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum DocType {
/// Long-term curated memory (MEMORY.md equivalent).
Memory,
/// Daily append-only logs.
DailyLog,
/// Agent identity (name, nature, vibe).
Identity,
/// Core values and principles (SOUL.md).
Soul,
/// Behavior instructions (AGENTS.md).
Agents,
/// User context (USER.md).
User,
/// Periodic checklist (HEARTBEAT.md).
Heartbeat,
}
impl DocType {
/// Get the string representation.
pub fn as_str(&self) -> &'static str {
match self {
DocType::Memory => "memory",
DocType::DailyLog => "daily_log",
DocType::Identity => "identity",
DocType::Soul => "soul",
DocType::Agents => "agents",
DocType::User => "user",
DocType::Heartbeat => "heartbeat",
}
}
/// Check if this document type is a singleton (one per user/agent).
pub fn is_singleton(&self) -> bool {
match self {
DocType::Memory
| DocType::Identity
| DocType::Soul
| DocType::Agents
| DocType::User
| DocType::Heartbeat => true,
DocType::DailyLog => false,
}
}
/// Check if this document should be included in the system prompt.
pub fn is_identity_document(&self) -> bool {
matches!(
self,
DocType::Identity | DocType::Soul | DocType::Agents | DocType::User
)
}
}
impl TryFrom<&str> for DocType {
type Error = WorkspaceError;
fn try_from(s: &str) -> Result<Self, Self::Error> {
match s {
"memory" => Ok(DocType::Memory),
"daily_log" => Ok(DocType::DailyLog),
"identity" => Ok(DocType::Identity),
"soul" => Ok(DocType::Soul),
"agents" => Ok(DocType::Agents),
"user" => Ok(DocType::User),
"heartbeat" => Ok(DocType::Heartbeat),
_ => Err(WorkspaceError::InvalidDocType {
doc_type: s.to_string(),
}),
}
}
}
impl std::fmt::Display for DocType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", self.as_str())
}
}
/// A memory document stored in the database.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryDocument {
/// Unique document ID.
pub id: Uuid,
/// User identifier.
pub user_id: String,
/// Optional agent ID for multi-agent isolation.
pub agent_id: Option<Uuid>,
/// Document type.
pub doc_type: DocType,
/// Optional title (e.g., date for daily logs).
pub title: Option<String>,
/// Full document content.
pub content: String,
/// Creation timestamp.
pub created_at: DateTime<Utc>,
/// Last update timestamp.
pub updated_at: DateTime<Utc>,
/// Flexible metadata.
pub metadata: serde_json::Value,
}
impl MemoryDocument {
/// Create a new document (not persisted yet).
pub fn new(
user_id: impl Into<String>,
agent_id: Option<Uuid>,
doc_type: DocType,
title: Option<String>,
) -> Self {
let now = Utc::now();
Self {
id: Uuid::new_v4(),
user_id: user_id.into(),
agent_id,
doc_type,
title,
content: String::new(),
created_at: now,
updated_at: now,
metadata: serde_json::Value::Object(serde_json::Map::new()),
}
}
/// Check if the document is empty.
pub fn is_empty(&self) -> bool {
self.content.is_empty()
}
/// Get word count.
pub fn word_count(&self) -> usize {
self.content.split_whitespace().count()
}
}
/// A chunk of a memory document for search indexing.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct MemoryChunk {
/// Unique chunk ID.
pub id: Uuid,
/// Parent document ID.
pub document_id: Uuid,
/// Position in the document (0-based).
pub chunk_index: i32,
/// Chunk text content.
pub content: String,
/// Embedding vector (if generated).
pub embedding: Option<Vec<f32>>,
/// Creation timestamp.
pub created_at: DateTime<Utc>,
}
impl MemoryChunk {
/// Create a new chunk (not persisted yet).
pub fn new(document_id: Uuid, chunk_index: i32, content: impl Into<String>) -> Self {
Self {
id: Uuid::new_v4(),
document_id,
chunk_index,
content: content.into(),
embedding: None,
created_at: Utc::now(),
}
}
/// Set the embedding.
pub fn with_embedding(mut self, embedding: Vec<f32>) -> Self {
self.embedding = Some(embedding);
self
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_doc_type_roundtrip() {
for doc_type in [
DocType::Memory,
DocType::DailyLog,
DocType::Identity,
DocType::Soul,
DocType::Agents,
DocType::User,
DocType::Heartbeat,
] {
let s = doc_type.as_str();
let parsed = DocType::try_from(s).unwrap();
assert_eq!(parsed, doc_type);
}
}
#[test]
fn test_singleton_types() {
assert!(DocType::Memory.is_singleton());
assert!(DocType::Heartbeat.is_singleton());
assert!(!DocType::DailyLog.is_singleton());
}
#[test]
fn test_identity_documents() {
assert!(DocType::Soul.is_identity_document());
assert!(DocType::Agents.is_identity_document());
assert!(!DocType::Memory.is_identity_document());
assert!(!DocType::DailyLog.is_identity_document());
}
#[test]
fn test_memory_document_word_count() {
let mut doc = MemoryDocument::new("user1", None, DocType::Memory, None);
assert_eq!(doc.word_count(), 0);
doc.content = "Hello world, this is a test.".to_string();
assert_eq!(doc.word_count(), 6);
}
}
+324
View File
@@ -0,0 +1,324 @@
//! Embedding providers for semantic search.
//!
//! Embeddings convert text into dense vectors that capture semantic meaning.
//! Similar concepts have similar vectors, enabling semantic search.
use async_trait::async_trait;
use serde::{Deserialize, Serialize};
/// Error type for embedding operations.
#[derive(Debug, thiserror::Error)]
pub enum EmbeddingError {
#[error("HTTP request failed: {0}")]
HttpError(String),
#[error("Invalid response: {0}")]
InvalidResponse(String),
#[error("Rate limited, retry after {retry_after:?}")]
RateLimited {
retry_after: Option<std::time::Duration>,
},
#[error("Authentication failed")]
AuthFailed,
#[error("Text too long: {length} > {max}")]
TextTooLong { length: usize, max: usize },
}
impl From<reqwest::Error> for EmbeddingError {
fn from(e: reqwest::Error) -> Self {
EmbeddingError::HttpError(e.to_string())
}
}
/// Trait for embedding providers.
#[async_trait]
pub trait EmbeddingProvider: Send + Sync {
/// Get the embedding dimension.
fn dimension(&self) -> usize;
/// Get the model name.
fn model_name(&self) -> &str;
/// Maximum input length in characters.
fn max_input_length(&self) -> usize;
/// Generate an embedding for a single text.
async fn embed(&self, text: &str) -> Result<Vec<f32>, EmbeddingError>;
/// Generate embeddings for multiple texts (batched).
///
/// Default implementation calls embed() for each text.
async fn embed_batch(&self, texts: &[String]) -> Result<Vec<Vec<f32>>, EmbeddingError> {
let mut embeddings = Vec::with_capacity(texts.len());
for text in texts {
embeddings.push(self.embed(text).await?);
}
Ok(embeddings)
}
}
/// OpenAI embedding provider using text-embedding-ada-002 or text-embedding-3-small.
pub struct OpenAiEmbeddings {
client: reqwest::Client,
api_key: String,
model: String,
dimension: usize,
}
impl OpenAiEmbeddings {
/// Create a new OpenAI embedding provider with the default model.
///
/// Uses text-embedding-3-small which has 1536 dimensions.
pub fn new(api_key: impl Into<String>) -> Self {
Self {
client: reqwest::Client::new(),
api_key: api_key.into(),
model: "text-embedding-3-small".to_string(),
dimension: 1536,
}
}
/// Use text-embedding-ada-002 model.
pub fn ada_002(api_key: impl Into<String>) -> Self {
Self {
client: reqwest::Client::new(),
api_key: api_key.into(),
model: "text-embedding-ada-002".to_string(),
dimension: 1536,
}
}
/// Use text-embedding-3-large model.
pub fn large(api_key: impl Into<String>) -> Self {
Self {
client: reqwest::Client::new(),
api_key: api_key.into(),
model: "text-embedding-3-large".to_string(),
dimension: 3072,
}
}
/// Use a custom model with specified dimension.
pub fn with_model(
api_key: impl Into<String>,
model: impl Into<String>,
dimension: usize,
) -> Self {
Self {
client: reqwest::Client::new(),
api_key: api_key.into(),
model: model.into(),
dimension,
}
}
}
#[derive(Debug, Serialize)]
struct OpenAiEmbeddingRequest<'a> {
model: &'a str,
input: &'a [String],
}
#[derive(Debug, Deserialize)]
struct OpenAiEmbeddingResponse {
data: Vec<OpenAiEmbeddingData>,
}
#[derive(Debug, Deserialize)]
struct OpenAiEmbeddingData {
embedding: Vec<f32>,
}
#[async_trait]
impl EmbeddingProvider for OpenAiEmbeddings {
fn dimension(&self) -> usize {
self.dimension
}
fn model_name(&self) -> &str {
&self.model
}
fn max_input_length(&self) -> usize {
// text-embedding-3-small/large: 8191 tokens (~32k chars)
// text-embedding-ada-002: 8191 tokens
32_000
}
async fn embed(&self, text: &str) -> Result<Vec<f32>, EmbeddingError> {
if text.len() > self.max_input_length() {
return Err(EmbeddingError::TextTooLong {
length: text.len(),
max: self.max_input_length(),
});
}
let embeddings = self.embed_batch(&[text.to_string()]).await?;
embeddings
.into_iter()
.next()
.ok_or_else(|| EmbeddingError::InvalidResponse("No embedding returned".to_string()))
}
async fn embed_batch(&self, texts: &[String]) -> Result<Vec<Vec<f32>>, EmbeddingError> {
if texts.is_empty() {
return Ok(Vec::new());
}
let request = OpenAiEmbeddingRequest {
model: &self.model,
input: texts,
};
let response = self
.client
.post("https://api.openai.com/v1/embeddings")
.header("Authorization", format!("Bearer {}", self.api_key))
.json(&request)
.send()
.await?;
let status = response.status();
if status == reqwest::StatusCode::UNAUTHORIZED {
return Err(EmbeddingError::AuthFailed);
}
if status == reqwest::StatusCode::TOO_MANY_REQUESTS {
let retry_after = response
.headers()
.get("retry-after")
.and_then(|v| v.to_str().ok())
.and_then(|s| s.parse::<u64>().ok())
.map(std::time::Duration::from_secs);
return Err(EmbeddingError::RateLimited { retry_after });
}
if !status.is_success() {
let error_text = response.text().await.unwrap_or_default();
return Err(EmbeddingError::HttpError(format!(
"Status {}: {}",
status, error_text
)));
}
let result: OpenAiEmbeddingResponse = response.json().await.map_err(|e| {
EmbeddingError::InvalidResponse(format!("Failed to parse response: {}", e))
})?;
Ok(result.data.into_iter().map(|d| d.embedding).collect())
}
}
/// A mock embedding provider for testing.
#[cfg(test)]
pub struct MockEmbeddings {
dimension: usize,
}
#[cfg(test)]
impl MockEmbeddings {
pub fn new(dimension: usize) -> Self {
Self { dimension }
}
}
#[cfg(test)]
#[async_trait]
impl EmbeddingProvider for MockEmbeddings {
fn dimension(&self) -> usize {
self.dimension
}
fn model_name(&self) -> &str {
"mock-embedding"
}
fn max_input_length(&self) -> usize {
10_000
}
async fn embed(&self, text: &str) -> Result<Vec<f32>, EmbeddingError> {
// Generate a deterministic embedding based on text hash
use std::hash::{Hash, Hasher};
let mut hasher = std::collections::hash_map::DefaultHasher::new();
text.hash(&mut hasher);
let hash = hasher.finish();
let mut embedding = Vec::with_capacity(self.dimension);
let mut seed = hash;
for _ in 0..self.dimension {
// Simple LCG for deterministic random values
seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1);
let value = (seed as f32 / u64::MAX as f32) * 2.0 - 1.0;
embedding.push(value);
}
// Normalize to unit length
let magnitude: f32 = embedding.iter().map(|x| x * x).sum::<f32>().sqrt();
if magnitude > 0.0 {
for x in &mut embedding {
*x /= magnitude;
}
}
Ok(embedding)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_mock_embeddings() {
let provider = MockEmbeddings::new(128);
let embedding = provider.embed("hello world").await.unwrap();
assert_eq!(embedding.len(), 128);
// Check normalization (should be unit vector)
let magnitude: f32 = embedding.iter().map(|x| x * x).sum::<f32>().sqrt();
assert!((magnitude - 1.0).abs() < 0.001);
}
#[tokio::test]
async fn test_mock_embeddings_deterministic() {
let provider = MockEmbeddings::new(64);
let emb1 = provider.embed("test").await.unwrap();
let emb2 = provider.embed("test").await.unwrap();
// Same input should produce same embedding
assert_eq!(emb1, emb2);
}
#[tokio::test]
async fn test_mock_embeddings_batch() {
let provider = MockEmbeddings::new(64);
let texts = vec!["hello".to_string(), "world".to_string()];
let embeddings = provider.embed_batch(&texts).await.unwrap();
assert_eq!(embeddings.len(), 2);
assert_eq!(embeddings[0].len(), 64);
assert_eq!(embeddings[1].len(), 64);
// Different texts should produce different embeddings
assert_ne!(embeddings[0], embeddings[1]);
}
#[test]
fn test_openai_embeddings_config() {
let provider = OpenAiEmbeddings::new("test-key");
assert_eq!(provider.dimension(), 1536);
assert_eq!(provider.model_name(), "text-embedding-3-small");
let provider = OpenAiEmbeddings::large("test-key");
assert_eq!(provider.dimension(), 3072);
assert_eq!(provider.model_name(), "text-embedding-3-large");
}
}
+392
View File
@@ -0,0 +1,392 @@
//! Workspace and memory system (OpenClaw-inspired).
//!
//! The workspace provides persistent memory for agents:
//! - **MEMORY.md**: Long-term curated memory (facts, decisions, preferences)
//! - **Daily logs**: Append-only daily notes (raw context)
//! - **Identity files**: Agent personality and user context
//! - **HEARTBEAT.md**: Periodic checklist for proactive execution
//!
//! Memory is searchable via hybrid search (FTS + semantic embeddings).
//!
//! # Architecture
//!
//! ```text
//! ┌─────────────────────────────────────────────────────────────┐
//! │ Workspace │
//! │ ┌────────────────┐ ┌────────────────┐ ┌──────────────┐ │
//! │ │ MemoryDocument │ │ MemoryChunk │ │ Search │ │
//! │ │ (full docs) │──│ (chunked) │──│ (FTS+vector) │ │
//! │ └────────────────┘ └────────────────┘ └──────────────┘ │
//! │ │ │ │ │
//! │ └───────────────────┴──────────────────┘ │
//! │ │ │
//! │ ┌──────┴──────┐ │
//! │ │ Repository │ │
//! │ │ (PostgreSQL)│ │
//! │ └─────────────┘ │
//! └─────────────────────────────────────────────────────────────┘
//! ```
//!
//! # Key Patterns
//!
//! 1. **Memory is persistence**: If you want to remember something, write it
//! 2. **Two-tier memory**: Daily logs (raw) + MEMORY.md (curated)
//! 3. **Hybrid search**: Vector similarity + BM25 full-text via RRF
mod chunker;
mod document;
mod embeddings;
mod repository;
mod search;
pub use chunker::{ChunkConfig, chunk_document};
pub use document::{DocType, MemoryChunk, MemoryDocument};
pub use embeddings::{EmbeddingProvider, OpenAiEmbeddings};
pub use repository::Repository;
pub use search::{SearchConfig, SearchResult};
use std::sync::Arc;
use chrono::{NaiveDate, Utc};
use deadpool_postgres::Pool;
use uuid::Uuid;
use crate::error::WorkspaceError;
/// Workspace provides database-backed memory storage for an agent.
///
/// Each workspace is scoped to a user (and optionally an agent).
/// Documents are persisted to PostgreSQL and indexed for search.
pub struct Workspace {
/// User identifier (from channel).
user_id: String,
/// Optional agent ID for multi-agent isolation.
agent_id: Option<Uuid>,
/// Database repository.
repo: Repository,
/// Embedding provider for semantic search.
embeddings: Option<Arc<dyn EmbeddingProvider>>,
}
impl Workspace {
/// Create a new workspace for a user.
pub fn new(user_id: impl Into<String>, pool: Pool) -> Self {
Self {
user_id: user_id.into(),
agent_id: None,
repo: Repository::new(pool),
embeddings: None,
}
}
/// Create a workspace with a specific agent ID.
pub fn with_agent(mut self, agent_id: Uuid) -> Self {
self.agent_id = Some(agent_id);
self
}
/// Set the embedding provider for semantic search.
pub fn with_embeddings(mut self, provider: Arc<dyn EmbeddingProvider>) -> Self {
self.embeddings = Some(provider);
self
}
/// Get the user ID.
pub fn user_id(&self) -> &str {
&self.user_id
}
/// Get the agent ID.
pub fn agent_id(&self) -> Option<Uuid> {
self.agent_id
}
// ==================== Document Access ====================
/// Get the main MEMORY.md document (long-term curated memory).
///
/// Creates it if it doesn't exist.
pub async fn memory(&self) -> Result<MemoryDocument, WorkspaceError> {
self.repo
.get_or_create_document(&self.user_id, self.agent_id, DocType::Memory, None)
.await
}
/// Get today's daily log.
///
/// Daily logs are append-only and keyed by date.
pub async fn today_log(&self) -> Result<MemoryDocument, WorkspaceError> {
let today = Utc::now().date_naive();
self.daily_log(today).await
}
/// Get a daily log for a specific date.
pub async fn daily_log(&self, date: NaiveDate) -> Result<MemoryDocument, WorkspaceError> {
let title = date.format("%Y-%m-%d").to_string();
self.repo
.get_or_create_document(
&self.user_id,
self.agent_id,
DocType::DailyLog,
Some(&title),
)
.await
}
/// Get the heartbeat checklist (HEARTBEAT.md).
pub async fn heartbeat_checklist(&self) -> Result<Option<String>, WorkspaceError> {
match self
.repo
.get_document(&self.user_id, self.agent_id, DocType::Heartbeat, None)
.await
{
Ok(doc) => Ok(Some(doc.content)),
Err(WorkspaceError::DocumentNotFound { .. }) => Ok(None),
Err(e) => Err(e),
}
}
/// Get a document by type.
pub async fn get_document(
&self,
doc_type: DocType,
title: Option<&str>,
) -> Result<MemoryDocument, WorkspaceError> {
self.repo
.get_document(&self.user_id, self.agent_id, doc_type, title)
.await
}
// ==================== Memory Operations ====================
/// Append an entry to the main MEMORY.md document.
///
/// This is for important facts, decisions, and preferences worth
/// remembering long-term.
pub async fn append_memory(&self, entry: &str) -> Result<(), WorkspaceError> {
let doc = self.memory().await?;
let new_content = if doc.content.is_empty() {
entry.to_string()
} else {
format!("{}\n\n{}", doc.content, entry)
};
self.repo.update_document(doc.id, &new_content).await?;
self.reindex_document(doc.id).await?;
Ok(())
}
/// Append an entry to today's daily log.
///
/// Daily logs are raw, append-only notes for the current day.
pub async fn append_daily_log(&self, entry: &str) -> Result<(), WorkspaceError> {
let doc = self.today_log().await?;
let timestamp = Utc::now().format("%H:%M:%S");
let timestamped_entry = format!("[{}] {}", timestamp, entry);
let new_content = if doc.content.is_empty() {
timestamped_entry
} else {
format!("{}\n{}", doc.content, timestamped_entry)
};
self.repo.update_document(doc.id, &new_content).await?;
self.reindex_document(doc.id).await?;
Ok(())
}
/// Update a document's content entirely.
pub async fn update_document(
&self,
doc_type: DocType,
title: Option<&str>,
content: &str,
) -> Result<(), WorkspaceError> {
let doc = self
.repo
.get_or_create_document(&self.user_id, self.agent_id, doc_type, title)
.await?;
self.repo.update_document(doc.id, content).await?;
self.reindex_document(doc.id).await?;
Ok(())
}
// ==================== System Prompt ====================
/// Build the system prompt from identity files.
///
/// Loads AGENTS.md, SOUL.md, USER.md, and IDENTITY.md to compose
/// the agent's system prompt.
pub async fn system_prompt(&self) -> Result<String, WorkspaceError> {
let mut parts = Vec::new();
// Load identity files in order of importance
let identity_types = [
(DocType::Agents, "## Agent Instructions"),
(DocType::Soul, "## Core Values"),
(DocType::User, "## User Context"),
(DocType::Identity, "## Identity"),
];
for (doc_type, header) in identity_types {
if let Ok(doc) = self
.repo
.get_document(&self.user_id, self.agent_id, doc_type, None)
.await
{
if !doc.content.is_empty() {
parts.push(format!("{}\n\n{}", header, doc.content));
}
}
}
// Add today's memory context (last 2 days of daily logs)
let today = Utc::now().date_naive();
let yesterday = today.pred_opt().unwrap_or(today);
for date in [today, yesterday] {
if let Ok(doc) = self.daily_log(date).await {
if !doc.content.is_empty() {
let header = if date == today {
"## Today's Notes"
} else {
"## Yesterday's Notes"
};
parts.push(format!("{}\n\n{}", header, doc.content));
}
}
}
Ok(parts.join("\n\n---\n\n"))
}
// ==================== Search ====================
/// Hybrid search across all memory documents.
///
/// Combines full-text search (BM25) with semantic search (vector similarity)
/// using Reciprocal Rank Fusion (RRF).
pub async fn search(
&self,
query: &str,
limit: usize,
) -> Result<Vec<SearchResult>, WorkspaceError> {
self.search_with_config(query, SearchConfig::default().with_limit(limit))
.await
}
/// Search with custom configuration.
pub async fn search_with_config(
&self,
query: &str,
config: SearchConfig,
) -> Result<Vec<SearchResult>, WorkspaceError> {
// Generate embedding for semantic search if provider available
let embedding = if let Some(ref provider) = self.embeddings {
Some(
provider
.embed(query)
.await
.map_err(|e| WorkspaceError::EmbeddingFailed {
reason: e.to_string(),
})?,
)
} else {
None
};
self.repo
.hybrid_search(
&self.user_id,
self.agent_id,
query,
embedding.as_deref(),
&config,
)
.await
}
// ==================== Indexing ====================
/// Re-index a document (chunk and generate embeddings).
async fn reindex_document(&self, document_id: Uuid) -> Result<(), WorkspaceError> {
// Get the document
let doc = self.repo.get_document_by_id(document_id).await?;
// Chunk the content
let chunks = chunk_document(&doc.content, ChunkConfig::default());
// Delete old chunks
self.repo.delete_chunks(document_id).await?;
// Insert new chunks
for (index, content) in chunks.into_iter().enumerate() {
// Generate embedding if provider available
let embedding = if let Some(ref provider) = self.embeddings {
match provider.embed(&content).await {
Ok(emb) => Some(emb),
Err(e) => {
tracing::warn!("Failed to generate embedding: {}", e);
None
}
}
} else {
None
};
self.repo
.insert_chunk(document_id, index as i32, &content, embedding.as_deref())
.await?;
}
Ok(())
}
/// Generate embeddings for chunks that don't have them yet.
///
/// This is useful for backfilling embeddings after enabling the provider.
pub async fn backfill_embeddings(&self) -> Result<usize, WorkspaceError> {
let Some(ref provider) = self.embeddings else {
return Ok(0);
};
let chunks = self
.repo
.get_chunks_without_embeddings(&self.user_id, self.agent_id, 100)
.await?;
let mut count = 0;
for chunk in chunks {
match provider.embed(&chunk.content).await {
Ok(embedding) => {
self.repo
.update_chunk_embedding(chunk.id, &embedding)
.await?;
count += 1;
}
Err(e) => {
tracing::warn!("Failed to embed chunk {}: {}", chunk.id, e);
}
}
}
Ok(count)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_doc_type_display() {
assert_eq!(DocType::Memory.as_str(), "memory");
assert_eq!(DocType::DailyLog.as_str(), "daily_log");
assert_eq!(DocType::Heartbeat.as_str(), "heartbeat");
}
#[test]
fn test_doc_type_parse() {
assert_eq!(DocType::try_from("memory").unwrap(), DocType::Memory);
assert_eq!(DocType::try_from("daily_log").unwrap(), DocType::DailyLog);
assert!(DocType::try_from("invalid").is_err());
}
}
+451
View File
@@ -0,0 +1,451 @@
//! Database repository for workspace persistence.
//!
//! All workspace data is stored in PostgreSQL:
//! - Documents in `memory_documents` table
//! - Chunks in `memory_chunks` table (with FTS and vector indexes)
use chrono::Utc;
use deadpool_postgres::Pool;
use pgvector::Vector;
use uuid::Uuid;
use crate::error::WorkspaceError;
use crate::workspace::document::{DocType, MemoryChunk, MemoryDocument};
use crate::workspace::search::{RankedResult, SearchConfig, SearchResult, reciprocal_rank_fusion};
/// Database repository for workspace operations.
pub struct Repository {
pool: Pool,
}
impl Repository {
/// Create a new repository with a connection pool.
pub fn new(pool: Pool) -> Self {
Self { pool }
}
/// Get a connection from the pool.
async fn conn(&self) -> Result<deadpool_postgres::Object, WorkspaceError> {
self.pool
.get()
.await
.map_err(|e| WorkspaceError::SearchFailed {
reason: format!("Failed to get connection: {}", e),
})
}
// ==================== Document Operations ====================
/// Get a document by type and optional title.
pub async fn get_document(
&self,
user_id: &str,
agent_id: Option<Uuid>,
doc_type: DocType,
title: Option<&str>,
) -> Result<MemoryDocument, WorkspaceError> {
let conn = self.conn().await?;
let row = if let Some(title) = title {
conn.query_opt(
r#"
SELECT id, user_id, agent_id, doc_type, title, content,
created_at, updated_at, metadata
FROM memory_documents
WHERE user_id = $1 AND agent_id IS NOT DISTINCT FROM $2
AND doc_type = $3 AND title = $4
"#,
&[&user_id, &agent_id, &doc_type.as_str(), &title],
)
.await
} else {
conn.query_opt(
r#"
SELECT id, user_id, agent_id, doc_type, title, content,
created_at, updated_at, metadata
FROM memory_documents
WHERE user_id = $1 AND agent_id IS NOT DISTINCT FROM $2
AND doc_type = $3 AND title IS NULL
"#,
&[&user_id, &agent_id, &doc_type.as_str()],
)
.await
};
let row = row.map_err(|e| WorkspaceError::SearchFailed {
reason: format!("Query failed: {}", e),
})?;
match row {
Some(row) => Ok(self.row_to_document(&row)?),
None => Err(WorkspaceError::DocumentNotFound {
doc_type: doc_type.to_string(),
user_id: user_id.to_string(),
}),
}
}
/// Get a document by ID.
pub async fn get_document_by_id(&self, id: Uuid) -> Result<MemoryDocument, WorkspaceError> {
let conn = self.conn().await?;
let row = conn
.query_opt(
r#"
SELECT id, user_id, agent_id, doc_type, title, content,
created_at, updated_at, metadata
FROM memory_documents WHERE id = $1
"#,
&[&id],
)
.await
.map_err(|e| WorkspaceError::SearchFailed {
reason: format!("Query failed: {}", e),
})?;
match row {
Some(row) => Ok(self.row_to_document(&row)?),
None => Err(WorkspaceError::DocumentNotFound {
doc_type: "unknown".to_string(),
user_id: "unknown".to_string(),
}),
}
}
/// Get or create a document.
pub async fn get_or_create_document(
&self,
user_id: &str,
agent_id: Option<Uuid>,
doc_type: DocType,
title: Option<&str>,
) -> Result<MemoryDocument, WorkspaceError> {
// Try to get existing document first
match self.get_document(user_id, agent_id, doc_type, title).await {
Ok(doc) => return Ok(doc),
Err(WorkspaceError::DocumentNotFound { .. }) => {}
Err(e) => return Err(e),
}
// Create new document
let conn = self.conn().await?;
let id = Uuid::new_v4();
let now = Utc::now();
conn.execute(
r#"
INSERT INTO memory_documents (id, user_id, agent_id, doc_type, title, content, created_at, updated_at)
VALUES ($1, $2, $3, $4, $5, '', $6, $7)
ON CONFLICT (user_id, agent_id, doc_type, title) DO NOTHING
"#,
&[&id, &user_id, &agent_id, &doc_type.as_str(), &title, &now, &now],
)
.await
.map_err(|e| WorkspaceError::SearchFailed {
reason: format!("Insert failed: {}", e),
})?;
// Fetch the document (might have been created by concurrent request)
self.get_document(user_id, agent_id, doc_type, title).await
}
/// Update a document's content.
pub async fn update_document(&self, id: Uuid, content: &str) -> Result<(), WorkspaceError> {
let conn = self.conn().await?;
conn.execute(
"UPDATE memory_documents SET content = $2, updated_at = NOW() WHERE id = $1",
&[&id, &content],
)
.await
.map_err(|e| WorkspaceError::SearchFailed {
reason: format!("Update failed: {}", e),
})?;
Ok(())
}
/// List documents by type.
pub async fn list_documents(
&self,
user_id: &str,
agent_id: Option<Uuid>,
doc_type: Option<DocType>,
) -> Result<Vec<MemoryDocument>, WorkspaceError> {
let conn = self.conn().await?;
let rows = if let Some(dt) = doc_type {
conn.query(
r#"
SELECT id, user_id, agent_id, doc_type, title, content,
created_at, updated_at, metadata
FROM memory_documents
WHERE user_id = $1 AND agent_id IS NOT DISTINCT FROM $2 AND doc_type = $3
ORDER BY updated_at DESC
"#,
&[&user_id, &agent_id, &dt.as_str()],
)
.await
} else {
conn.query(
r#"
SELECT id, user_id, agent_id, doc_type, title, content,
created_at, updated_at, metadata
FROM memory_documents
WHERE user_id = $1 AND agent_id IS NOT DISTINCT FROM $2
ORDER BY updated_at DESC
"#,
&[&user_id, &agent_id],
)
.await
};
let rows = rows.map_err(|e| WorkspaceError::SearchFailed {
reason: format!("Query failed: {}", e),
})?;
rows.iter().map(|r| self.row_to_document(r)).collect()
}
fn row_to_document(&self, row: &tokio_postgres::Row) -> Result<MemoryDocument, WorkspaceError> {
let doc_type_str: String = row.get("doc_type");
let doc_type = DocType::try_from(doc_type_str.as_str())?;
Ok(MemoryDocument {
id: row.get("id"),
user_id: row.get("user_id"),
agent_id: row.get("agent_id"),
doc_type,
title: row.get("title"),
content: row.get("content"),
created_at: row.get("created_at"),
updated_at: row.get("updated_at"),
metadata: row.get("metadata"),
})
}
// ==================== Chunk Operations ====================
/// Delete all chunks for a document.
pub async fn delete_chunks(&self, document_id: Uuid) -> Result<(), WorkspaceError> {
let conn = self.conn().await?;
conn.execute(
"DELETE FROM memory_chunks WHERE document_id = $1",
&[&document_id],
)
.await
.map_err(|e| WorkspaceError::ChunkingFailed {
reason: format!("Delete failed: {}", e),
})?;
Ok(())
}
/// Insert a chunk.
pub async fn insert_chunk(
&self,
document_id: Uuid,
chunk_index: i32,
content: &str,
embedding: Option<&[f32]>,
) -> Result<Uuid, WorkspaceError> {
let conn = self.conn().await?;
let id = Uuid::new_v4();
let embedding_vec = embedding.map(|e| Vector::from(e.to_vec()));
conn.execute(
r#"
INSERT INTO memory_chunks (id, document_id, chunk_index, content, embedding)
VALUES ($1, $2, $3, $4, $5)
"#,
&[&id, &document_id, &chunk_index, &content, &embedding_vec],
)
.await
.map_err(|e| WorkspaceError::ChunkingFailed {
reason: format!("Insert failed: {}", e),
})?;
Ok(id)
}
/// Update a chunk's embedding.
pub async fn update_chunk_embedding(
&self,
chunk_id: Uuid,
embedding: &[f32],
) -> Result<(), WorkspaceError> {
let conn = self.conn().await?;
let embedding_vec = Vector::from(embedding.to_vec());
conn.execute(
"UPDATE memory_chunks SET embedding = $2 WHERE id = $1",
&[&chunk_id, &embedding_vec],
)
.await
.map_err(|e| WorkspaceError::EmbeddingFailed {
reason: format!("Update failed: {}", e),
})?;
Ok(())
}
/// Get chunks without embeddings for backfilling.
pub async fn get_chunks_without_embeddings(
&self,
user_id: &str,
agent_id: Option<Uuid>,
limit: usize,
) -> Result<Vec<MemoryChunk>, WorkspaceError> {
let conn = self.conn().await?;
let rows = conn
.query(
r#"
SELECT c.id, c.document_id, c.chunk_index, c.content, c.created_at
FROM memory_chunks c
JOIN memory_documents d ON d.id = c.document_id
WHERE d.user_id = $1 AND d.agent_id IS NOT DISTINCT FROM $2
AND c.embedding IS NULL
LIMIT $3
"#,
&[&user_id, &agent_id, &(limit as i64)],
)
.await
.map_err(|e| WorkspaceError::SearchFailed {
reason: format!("Query failed: {}", e),
})?;
Ok(rows
.iter()
.map(|row| MemoryChunk {
id: row.get("id"),
document_id: row.get("document_id"),
chunk_index: row.get("chunk_index"),
content: row.get("content"),
embedding: None,
created_at: row.get("created_at"),
})
.collect())
}
// ==================== Search Operations ====================
/// Perform hybrid search combining FTS and vector similarity.
pub async fn hybrid_search(
&self,
user_id: &str,
agent_id: Option<Uuid>,
query: &str,
embedding: Option<&[f32]>,
config: &SearchConfig,
) -> Result<Vec<SearchResult>, WorkspaceError> {
let fts_results = if config.use_fts {
self.fts_search(user_id, agent_id, query, config.pre_fusion_limit)
.await?
} else {
Vec::new()
};
let vector_results = if config.use_vector && embedding.is_some() {
self.vector_search(
user_id,
agent_id,
embedding.unwrap(),
config.pre_fusion_limit,
)
.await?
} else {
Vec::new()
};
Ok(reciprocal_rank_fusion(fts_results, vector_results, config))
}
/// Full-text search using PostgreSQL ts_rank_cd.
async fn fts_search(
&self,
user_id: &str,
agent_id: Option<Uuid>,
query: &str,
limit: usize,
) -> Result<Vec<RankedResult>, WorkspaceError> {
let conn = self.conn().await?;
// Use plainto_tsquery for natural language queries
let rows = conn
.query(
r#"
SELECT c.id as chunk_id, c.document_id, c.content,
ts_rank_cd(c.content_tsv, plainto_tsquery('english', $3)) as rank
FROM memory_chunks c
JOIN memory_documents d ON d.id = c.document_id
WHERE d.user_id = $1 AND d.agent_id IS NOT DISTINCT FROM $2
AND c.content_tsv @@ plainto_tsquery('english', $3)
ORDER BY rank DESC
LIMIT $4
"#,
&[&user_id, &agent_id, &query, &(limit as i64)],
)
.await
.map_err(|e| WorkspaceError::SearchFailed {
reason: format!("FTS query failed: {}", e),
})?;
Ok(rows
.iter()
.enumerate()
.map(|(i, row)| RankedResult {
chunk_id: row.get("chunk_id"),
document_id: row.get("document_id"),
content: row.get("content"),
rank: (i + 1) as u32, // 1-based rank
})
.collect())
}
/// Vector similarity search using pgvector cosine distance.
async fn vector_search(
&self,
user_id: &str,
agent_id: Option<Uuid>,
embedding: &[f32],
limit: usize,
) -> Result<Vec<RankedResult>, WorkspaceError> {
let conn = self.conn().await?;
let embedding_vec = Vector::from(embedding.to_vec());
// Use cosine distance (<=>)
let rows = conn
.query(
r#"
SELECT c.id as chunk_id, c.document_id, c.content,
1 - (c.embedding <=> $3) as similarity
FROM memory_chunks c
JOIN memory_documents d ON d.id = c.document_id
WHERE d.user_id = $1 AND d.agent_id IS NOT DISTINCT FROM $2
AND c.embedding IS NOT NULL
ORDER BY c.embedding <=> $3
LIMIT $4
"#,
&[&user_id, &agent_id, &embedding_vec, &(limit as i64)],
)
.await
.map_err(|e| WorkspaceError::SearchFailed {
reason: format!("Vector query failed: {}", e),
})?;
Ok(rows
.iter()
.enumerate()
.map(|(i, row)| RankedResult {
chunk_id: row.get("chunk_id"),
document_id: row.get("document_id"),
content: row.get("content"),
rank: (i + 1) as u32, // 1-based rank
})
.collect())
}
}
+391
View File
@@ -0,0 +1,391 @@
//! Hybrid search combining full-text and semantic search.
//!
//! Uses Reciprocal Rank Fusion (RRF) to combine results from:
//! 1. PostgreSQL full-text search (ts_rank_cd)
//! 2. pgvector cosine similarity search
//!
//! RRF formula: score = sum(1 / (k + rank)) for each retrieval method
//! This is robust to different score scales and produces better results
//! than simple score averaging.
use std::collections::HashMap;
use uuid::Uuid;
/// Configuration for hybrid search.
#[derive(Debug, Clone)]
pub struct SearchConfig {
/// Maximum number of results to return.
pub limit: usize,
/// RRF constant (typically 60). Higher values favor top results more.
pub rrf_k: u32,
/// Whether to include FTS results.
pub use_fts: bool,
/// Whether to include vector results.
pub use_vector: bool,
/// Minimum score threshold (0.0-1.0).
pub min_score: f32,
/// Maximum results to fetch from each method before fusion.
pub pre_fusion_limit: usize,
}
impl Default for SearchConfig {
fn default() -> Self {
Self {
limit: 10,
rrf_k: 60,
use_fts: true,
use_vector: true,
min_score: 0.0,
pre_fusion_limit: 50,
}
}
}
impl SearchConfig {
/// Set the result limit.
pub fn with_limit(mut self, limit: usize) -> Self {
self.limit = limit;
self
}
/// Set the RRF constant.
pub fn with_rrf_k(mut self, k: u32) -> Self {
self.rrf_k = k;
self
}
/// Disable FTS (only use vector search).
pub fn vector_only(mut self) -> Self {
self.use_fts = false;
self.use_vector = true;
self
}
/// Disable vector search (only use FTS).
pub fn fts_only(mut self) -> Self {
self.use_fts = true;
self.use_vector = false;
self
}
/// Set minimum score threshold.
pub fn with_min_score(mut self, score: f32) -> Self {
self.min_score = score.clamp(0.0, 1.0);
self
}
}
/// A search result with hybrid scoring.
#[derive(Debug, Clone)]
pub struct SearchResult {
/// Document ID containing this chunk.
pub document_id: Uuid,
/// Chunk ID.
pub chunk_id: Uuid,
/// Chunk content.
pub content: String,
/// Combined RRF score (0.0-1.0 normalized).
pub score: f32,
/// Rank in FTS results (1-based, None if not in FTS results).
pub fts_rank: Option<u32>,
/// Rank in vector results (1-based, None if not in vector results).
pub vector_rank: Option<u32>,
}
impl SearchResult {
/// Check if this result came from FTS.
pub fn from_fts(&self) -> bool {
self.fts_rank.is_some()
}
/// Check if this result came from vector search.
pub fn from_vector(&self) -> bool {
self.vector_rank.is_some()
}
/// Check if this result came from both methods (hybrid match).
pub fn is_hybrid(&self) -> bool {
self.fts_rank.is_some() && self.vector_rank.is_some()
}
}
/// Raw result from a single search method.
#[derive(Debug, Clone)]
pub struct RankedResult {
pub chunk_id: Uuid,
pub document_id: Uuid,
pub content: String,
pub rank: u32, // 1-based rank
}
/// Reciprocal Rank Fusion algorithm.
///
/// Combines ranked results from multiple retrieval methods using the formula:
/// score(d) = sum(1 / (k + rank(d))) for each method where d appears
///
/// # Arguments
///
/// * `fts_results` - Results from full-text search, ordered by relevance
/// * `vector_results` - Results from vector search, ordered by similarity
/// * `config` - Search configuration
///
/// # Returns
///
/// Combined results sorted by RRF score (descending).
pub fn reciprocal_rank_fusion(
fts_results: Vec<RankedResult>,
vector_results: Vec<RankedResult>,
config: &SearchConfig,
) -> Vec<SearchResult> {
let k = config.rrf_k as f32;
// Track scores and metadata for each chunk
struct ChunkInfo {
document_id: Uuid,
content: String,
score: f32,
fts_rank: Option<u32>,
vector_rank: Option<u32>,
}
let mut chunk_scores: HashMap<Uuid, ChunkInfo> = HashMap::new();
// Process FTS results
for result in fts_results {
let rrf_score = 1.0 / (k + result.rank as f32);
chunk_scores
.entry(result.chunk_id)
.and_modify(|info| {
info.score += rrf_score;
info.fts_rank = Some(result.rank);
})
.or_insert(ChunkInfo {
document_id: result.document_id,
content: result.content,
score: rrf_score,
fts_rank: Some(result.rank),
vector_rank: None,
});
}
// Process vector results
for result in vector_results {
let rrf_score = 1.0 / (k + result.rank as f32);
chunk_scores
.entry(result.chunk_id)
.and_modify(|info| {
info.score += rrf_score;
info.vector_rank = Some(result.rank);
})
.or_insert(ChunkInfo {
document_id: result.document_id,
content: result.content,
score: rrf_score,
fts_rank: None,
vector_rank: Some(result.rank),
});
}
// Convert to SearchResult and sort by score
let mut results: Vec<SearchResult> = chunk_scores
.into_iter()
.map(|(chunk_id, info)| SearchResult {
document_id: info.document_id,
chunk_id,
content: info.content,
score: info.score,
fts_rank: info.fts_rank,
vector_rank: info.vector_rank,
})
.collect();
// Normalize scores to 0-1 range
if let Some(max_score) = results.iter().map(|r| r.score).reduce(f32::max) {
if max_score > 0.0 {
for result in &mut results {
result.score /= max_score;
}
}
}
// Filter by minimum score
if config.min_score > 0.0 {
results.retain(|r| r.score >= config.min_score);
}
// Sort by score descending
results.sort_by(|a, b| {
b.score
.partial_cmp(&a.score)
.unwrap_or(std::cmp::Ordering::Equal)
});
// Limit results
results.truncate(config.limit);
results
}
#[cfg(test)]
mod tests {
use super::*;
fn make_result(chunk_id: Uuid, doc_id: Uuid, rank: u32) -> RankedResult {
RankedResult {
chunk_id,
document_id: doc_id,
content: format!("content for chunk {}", chunk_id),
rank,
}
}
#[test]
fn test_rrf_single_method() {
let config = SearchConfig::default().with_limit(10);
let chunk1 = Uuid::new_v4();
let chunk2 = Uuid::new_v4();
let doc = Uuid::new_v4();
let fts_results = vec![make_result(chunk1, doc, 1), make_result(chunk2, doc, 2)];
let results = reciprocal_rank_fusion(fts_results, Vec::new(), &config);
assert_eq!(results.len(), 2);
// First result should have higher score
assert!(results[0].score > results[1].score);
// All should have FTS rank
assert!(results.iter().all(|r| r.fts_rank.is_some()));
assert!(results.iter().all(|r| r.vector_rank.is_none()));
}
#[test]
fn test_rrf_hybrid_match_boosted() {
let config = SearchConfig::default().with_limit(10);
let chunk1 = Uuid::new_v4(); // In both
let chunk2 = Uuid::new_v4(); // FTS only
let chunk3 = Uuid::new_v4(); // Vector only
let doc = Uuid::new_v4();
let fts_results = vec![make_result(chunk1, doc, 1), make_result(chunk2, doc, 2)];
let vector_results = vec![make_result(chunk1, doc, 1), make_result(chunk3, doc, 2)];
let results = reciprocal_rank_fusion(fts_results, vector_results, &config);
assert_eq!(results.len(), 3);
// chunk1 should be first (hybrid match)
assert_eq!(results[0].chunk_id, chunk1);
assert!(results[0].is_hybrid());
assert!(results[0].score > results[1].score);
// Other chunks should not be hybrid
assert!(!results[1].is_hybrid());
assert!(!results[2].is_hybrid());
}
#[test]
fn test_rrf_score_normalization() {
let config = SearchConfig::default();
let chunk1 = Uuid::new_v4();
let doc = Uuid::new_v4();
let fts_results = vec![make_result(chunk1, doc, 1)];
let results = reciprocal_rank_fusion(fts_results, Vec::new(), &config);
// Single result should have normalized score of 1.0
assert_eq!(results.len(), 1);
assert!((results[0].score - 1.0).abs() < 0.001);
}
#[test]
fn test_rrf_min_score_filter() {
let config = SearchConfig::default().with_limit(10).with_min_score(0.5);
let chunk1 = Uuid::new_v4();
let chunk2 = Uuid::new_v4();
let chunk3 = Uuid::new_v4();
let doc = Uuid::new_v4();
// chunk1 has rank 1, chunk3 has rank 100 (low score)
let fts_results = vec![
make_result(chunk1, doc, 1),
make_result(chunk2, doc, 50),
make_result(chunk3, doc, 100),
];
let results = reciprocal_rank_fusion(fts_results, Vec::new(), &config);
// Low-scoring results should be filtered out
// All results should have score >= 0.5
for result in &results {
assert!(result.score >= 0.5);
}
}
#[test]
fn test_rrf_limit() {
let config = SearchConfig::default().with_limit(2);
let doc = Uuid::new_v4();
let fts_results: Vec<_> = (1..=5)
.map(|i| make_result(Uuid::new_v4(), doc, i))
.collect();
let results = reciprocal_rank_fusion(fts_results, Vec::new(), &config);
assert_eq!(results.len(), 2);
}
#[test]
fn test_rrf_k_parameter() {
// Higher k values make ranking differences less pronounced
let chunk1 = Uuid::new_v4();
let chunk2 = Uuid::new_v4();
let doc = Uuid::new_v4();
let fts_results = vec![make_result(chunk1, doc, 1), make_result(chunk2, doc, 2)];
// Low k: rank 1 score = 1/(10+1) = 0.091, rank 2 = 1/(10+2) = 0.083
let config_low_k = SearchConfig::default().with_rrf_k(10);
let results_low = reciprocal_rank_fusion(fts_results.clone(), Vec::new(), &config_low_k);
// High k: rank 1 score = 1/(100+1) = 0.0099, rank 2 = 1/(100+2) = 0.0098
let config_high_k = SearchConfig::default().with_rrf_k(100);
let results_high = reciprocal_rank_fusion(fts_results, Vec::new(), &config_high_k);
// With low k, the score difference is larger (relatively)
let diff_low = results_low[0].score - results_low[1].score;
let diff_high = results_high[0].score - results_high[1].score;
// Low k should have larger relative difference
assert!(diff_low > diff_high);
}
#[test]
fn test_search_config_builders() {
let config = SearchConfig::default()
.with_limit(20)
.with_rrf_k(30)
.with_min_score(0.1);
assert_eq!(config.limit, 20);
assert_eq!(config.rrf_k, 30);
assert!((config.min_score - 0.1).abs() < 0.001);
assert!(config.use_fts);
assert!(config.use_vector);
let fts_only = SearchConfig::default().fts_only();
assert!(fts_only.use_fts);
assert!(!fts_only.use_vector);
let vector_only = SearchConfig::default().vector_only();
assert!(!vector_only.use_fts);
assert!(vector_only.use_vector);
}
}