gitops/optimclaw
1
0
Fork 0
mirror of https://github.com/outbackdingo/optimclaw.git synced 2026-08-25 14:53:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Simplify workspace to path-based storage, remove legacy code

Browse Source 
- Consolidate all migrations into V1__initial.sql
- Replace DocType enum with flexible path-based file storage
- Add list_workspace_files SQL function for directory listing
- Update memory tools for path-based API (memory_read, memory_write,
  memory_search, memory_list)
- Remove unused OpenAI/Anthropic providers (NEAR AI only)
- Simplify config to remove multi-provider support
- Update CLAUDE.md documentation

Co-Authored-By: Claude Opus 4.5 <[email protected]>
This commit is contained in:
Illia Polosukhin
2026-02-02 21:38:53 -08:00
co-authored by Claude Opus 4.5
parent f29892b3fb
commit 3718cfa767
13 changed files with 877 additions and 1427 deletions
Download Patch File Download Diff File Expand all files Collapse all files
migrations/V1__initial.sql
+207 -9
View File
@@ -1,7 +1,12 @@
-- NEAR Agent Database Schema
-- V1: Initial schema
-- V1: Complete schema with workspace and memory system
-- Enable pgvector extension for semantic search
-- NOTE: Requires pgvector to be installed on PostgreSQL server
CREATE EXTENSION IF NOT EXISTS vector;
-- ==================== Conversations ====================
-- Conversations from various channels
CREATE TABLE conversations (
id UUID PRIMARY KEY,
channel TEXT NOT NULL,
@@ -16,7 +21,6 @@ CREATE INDEX idx_conversations_channel ON conversations(channel);
CREATE INDEX idx_conversations_user ON conversations(user_id);
CREATE INDEX idx_conversations_last_activity ON conversations(last_activity);
-- Messages in conversations
CREATE TABLE conversation_messages (
id UUID PRIMARY KEY,
conversation_id UUID NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
@@ -27,7 +31,8 @@ CREATE TABLE conversation_messages (
CREATE INDEX idx_conversation_messages_conversation ON conversation_messages(conversation_id);
-- Jobs we've worked on
-- ==================== Agent Jobs ====================
CREATE TABLE agent_jobs (
id UUID PRIMARY KEY,
marketplace_job_id UUID,
@@ -59,7 +64,6 @@ CREATE INDEX idx_agent_jobs_marketplace ON agent_jobs(marketplace_job_id);
CREATE INDEX idx_agent_jobs_conversation ON agent_jobs(conversation_id);
CREATE INDEX idx_agent_jobs_stuck ON agent_jobs(stuck_since) WHERE stuck_since IS NOT NULL;
-- Actions taken during job execution (event sourcing)
CREATE TABLE job_actions (
id UUID PRIMARY KEY,
job_id UUID NOT NULL REFERENCES agent_jobs(id) ON DELETE CASCADE,
@@ -80,7 +84,8 @@ CREATE TABLE job_actions (
CREATE INDEX idx_job_actions_job_id ON job_actions(job_id);
CREATE INDEX idx_job_actions_tool ON job_actions(tool_name);
-- Dynamic tools built by the agent
-- ==================== Dynamic Tools ====================
CREATE TABLE dynamic_tools (
id UUID PRIMARY KEY,
name TEXT NOT NULL UNIQUE,
@@ -100,7 +105,8 @@ CREATE TABLE dynamic_tools (
CREATE INDEX idx_dynamic_tools_status ON dynamic_tools(status);
CREATE INDEX idx_dynamic_tools_name ON dynamic_tools(name);
-- LLM calls for cost tracking
-- ==================== LLM Calls ====================
CREATE TABLE llm_calls (
id UUID PRIMARY KEY,
job_id UUID REFERENCES agent_jobs(id) ON DELETE CASCADE,
@@ -118,7 +124,8 @@ CREATE INDEX idx_llm_calls_job ON llm_calls(job_id);
CREATE INDEX idx_llm_calls_conversation ON llm_calls(conversation_id);
CREATE INDEX idx_llm_calls_provider ON llm_calls(provider);
-- Estimation history for continuous learning
-- ==================== Estimation ====================
CREATE TABLE estimation_snapshots (
id UUID PRIMARY KEY,
job_id UUID NOT NULL REFERENCES agent_jobs(id) ON DELETE CASCADE,
@@ -136,7 +143,8 @@ CREATE TABLE estimation_snapshots (
CREATE INDEX idx_estimation_category ON estimation_snapshots(category);
CREATE INDEX idx_estimation_job ON estimation_snapshots(job_id);
-- Self-repair history
-- ==================== Self Repair ====================
CREATE TABLE repair_attempts (
id UUID PRIMARY KEY,
target_type TEXT NOT NULL,
@@ -150,3 +158,193 @@ CREATE TABLE repair_attempts (
CREATE INDEX idx_repair_attempts_target ON repair_attempts(target_type, target_id);
CREATE INDEX idx_repair_attempts_created ON repair_attempts(created_at);
-- ==================== Workspace: Memory Documents ====================
-- Flexible filesystem-like structure for agent memory.
-- Agents can create arbitrary paths like:
-- "README.md", "context/vision.md", "daily/2024-01-15.md", "projects/alpha/notes.md"
CREATE TABLE memory_documents (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id TEXT NOT NULL,
agent_id UUID, -- NULL = shared across all agents for this user
-- File path within workspace (e.g., "context/vision.md")
path TEXT NOT NULL,
content TEXT NOT NULL,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
metadata JSONB NOT NULL DEFAULT '{}',
CONSTRAINT unique_path_per_user UNIQUE (user_id, agent_id, path)
);
CREATE INDEX idx_memory_documents_user ON memory_documents(user_id);
CREATE INDEX idx_memory_documents_path ON memory_documents(user_id, path);
CREATE INDEX idx_memory_documents_path_prefix ON memory_documents(user_id, path text_pattern_ops);
CREATE INDEX idx_memory_documents_updated ON memory_documents(updated_at DESC);
-- ==================== Workspace: Memory Chunks ====================
-- Documents are chunked for hybrid search (FTS + vector)
CREATE TABLE memory_chunks (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
document_id UUID NOT NULL REFERENCES memory_documents(id) ON DELETE CASCADE,
chunk_index INT NOT NULL,
content TEXT NOT NULL,
-- Full-text search vector
content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('english', content)) STORED,
-- Semantic search embedding (text-embedding-3-small = 1536 dims)
embedding VECTOR(1536),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
CONSTRAINT unique_chunk_per_doc UNIQUE (document_id, chunk_index)
);
CREATE INDEX idx_memory_chunks_tsv ON memory_chunks USING GIN(content_tsv);
CREATE INDEX idx_memory_chunks_embedding ON memory_chunks
USING hnsw(embedding vector_cosine_ops)
WITH (m = 16, ef_construction = 64);
CREATE INDEX idx_memory_chunks_document ON memory_chunks(document_id);
-- ==================== Workspace: Heartbeat State ====================
CREATE TABLE heartbeat_state (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id TEXT NOT NULL,
agent_id UUID,
last_run TIMESTAMPTZ,
next_run TIMESTAMPTZ,
interval_seconds INT NOT NULL DEFAULT 1800,
enabled BOOLEAN NOT NULL DEFAULT true,
consecutive_failures INT NOT NULL DEFAULT 0,
last_checks JSONB NOT NULL DEFAULT '{}',
CONSTRAINT unique_heartbeat_per_user UNIQUE (user_id, agent_id)
);
CREATE INDEX idx_heartbeat_user ON heartbeat_state(user_id);
CREATE INDEX idx_heartbeat_next_run ON heartbeat_state(next_run) WHERE enabled = true;
-- ==================== Helper Functions ====================
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = NOW();
RETURN NEW;
END;
$$ language 'plpgsql';
CREATE TRIGGER update_memory_documents_updated_at
BEFORE UPDATE ON memory_documents
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- Function to list files in a directory (prefix match)
CREATE OR REPLACE FUNCTION list_workspace_files(
p_user_id TEXT,
p_agent_id UUID,
p_directory TEXT DEFAULT ''
)
RETURNS TABLE (
path TEXT,
is_directory BOOLEAN,
updated_at TIMESTAMPTZ,
content_preview TEXT
) AS $$
BEGIN
-- Normalize directory path (ensure trailing slash for non-root)
IF p_directory != '' AND NOT p_directory LIKE '%/' THEN
p_directory := p_directory || '/';
END IF;
RETURN QUERY
WITH files AS (
SELECT
d.path,
d.updated_at,
LEFT(d.content, 200) as content_preview,
-- Extract the immediate child name
CASE
WHEN p_directory = '' THEN
CASE
WHEN position('/' in d.path) > 0
THEN substring(d.path from 1 for position('/' in d.path) - 1)
ELSE d.path
END
ELSE
CASE
WHEN position('/' in substring(d.path from length(p_directory) + 1)) > 0
THEN substring(
substring(d.path from length(p_directory) + 1)
from 1
for position('/' in substring(d.path from length(p_directory) + 1)) - 1
)
ELSE substring(d.path from length(p_directory) + 1)
END
END as child_name
FROM memory_documents d
WHERE d.user_id = p_user_id
AND d.agent_id IS NOT DISTINCT FROM p_agent_id
AND (p_directory = '' OR d.path LIKE p_directory || '%')
)
SELECT DISTINCT ON (f.child_name)
CASE
WHEN p_directory = '' THEN f.child_name
ELSE p_directory || f.child_name
END as path,
EXISTS (
SELECT 1 FROM memory_documents d2
WHERE d2.user_id = p_user_id
AND d2.agent_id IS NOT DISTINCT FROM p_agent_id
AND d2.path LIKE
CASE WHEN p_directory = '' THEN f.child_name ELSE p_directory || f.child_name END
|| '/%'
) as is_directory,
MAX(f.updated_at) as updated_at,
CASE
WHEN EXISTS (
SELECT 1 FROM memory_documents d2
WHERE d2.user_id = p_user_id
AND d2.agent_id IS NOT DISTINCT FROM p_agent_id
AND d2.path LIKE
CASE WHEN p_directory = '' THEN f.child_name ELSE p_directory || f.child_name END
|| '/%'
) THEN NULL
ELSE MAX(f.content_preview)
END as content_preview
FROM files f
WHERE f.child_name != '' AND f.child_name IS NOT NULL
GROUP BY f.child_name
ORDER BY f.child_name, is_directory DESC;
END;
$$ LANGUAGE plpgsql;
-- ==================== Views ====================
CREATE VIEW memory_documents_summary AS
SELECT
d.id,
d.user_id,
d.path,
d.created_at,
d.updated_at,
COUNT(c.id) as chunk_count,
COUNT(c.embedding) as embedded_chunk_count
FROM memory_documents d
LEFT JOIN memory_chunks c ON c.document_id = d.id
GROUP BY d.id;
CREATE VIEW chunks_pending_embedding AS
SELECT
c.id as chunk_id,
c.document_id,
d.user_id,
d.path,
LENGTH(c.content) as content_length
FROM memory_chunks c
JOIN memory_documents d ON d.id = c.document_id
WHERE c.embedding IS NULL;
migrations/V2__workspace_memory.sql
-159
View File
@@ -1,159 +0,0 @@
-- NEAR Agent Database Schema
-- V2: Workspace and memory system (OpenClaw-inspired)
--
-- This migration adds:
-- 1. Persistent memory documents (MEMORY.md, daily logs, identity files)
-- 2. Chunked content for hybrid search (FTS + vector)
-- 3. Heartbeat state for proactive execution
-- Enable pgvector extension for semantic search
-- NOTE: This requires pgvector to be installed on the PostgreSQL server
-- Install via: CREATE EXTENSION vector; (requires superuser or rds_superuser)
CREATE EXTENSION IF NOT EXISTS vector;
-- ==================== Memory Documents ====================
-- Stores full documents like MEMORY.md, daily logs, identity files
-- Think of this as the filesystem equivalent, but in PostgreSQL
CREATE TABLE memory_documents (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-- Ownership: who this document belongs to
user_id TEXT NOT NULL, -- User identifier (from channel)
agent_id UUID, -- NULL = shared across all agents for this user
-- Document type and content
doc_type TEXT NOT NULL, -- 'memory', 'daily_log', 'identity', 'soul', 'agents', 'user', 'heartbeat'
title TEXT, -- Optional title (e.g., date for daily logs)
content TEXT NOT NULL, -- Full document content
-- Timestamps
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-- Flexible metadata (tags, source, etc.)
metadata JSONB NOT NULL DEFAULT '{}',
-- Ensure one document per type per user (for singleton docs like MEMORY.md)
-- Daily logs use title as the date discriminator
CONSTRAINT unique_doc_per_user_type UNIQUE (user_id, agent_id, doc_type, title)
);
-- Indexes for common queries
CREATE INDEX idx_memory_documents_user ON memory_documents(user_id);
CREATE INDEX idx_memory_documents_user_type ON memory_documents(user_id, doc_type);
CREATE INDEX idx_memory_documents_updated ON memory_documents(updated_at DESC);
-- ==================== Memory Chunks ====================
-- Documents are chunked for search. Each chunk has:
-- 1. Full-text search vector (tsvector) for keyword matching
-- 2. Embedding vector for semantic similarity
--
-- Hybrid search combines both using Reciprocal Rank Fusion (RRF)
CREATE TABLE memory_chunks (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
document_id UUID NOT NULL REFERENCES memory_documents(id) ON DELETE CASCADE,
-- Chunk position and content
chunk_index INT NOT NULL, -- Position in document (0-based)
content TEXT NOT NULL, -- Chunk text (~800 tokens with 15% overlap)
-- Full-text search: auto-generated tsvector
content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('english', content)) STORED,
-- Semantic search: embedding vector (OpenAI text-embedding-ada-002 = 1536 dims)
-- NULL until embeddings are generated
embedding VECTOR(1536),
-- Timestamps
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-- Each chunk index unique per document
CONSTRAINT unique_chunk_per_doc UNIQUE (document_id, chunk_index)
);
-- GIN index for full-text search
CREATE INDEX idx_memory_chunks_tsv ON memory_chunks USING GIN(content_tsv);
-- HNSW index for vector similarity (cosine distance)
-- HNSW is faster than IVFFlat for reads, slightly slower for writes
CREATE INDEX idx_memory_chunks_embedding ON memory_chunks
USING hnsw(embedding vector_cosine_ops)
WITH (m = 16, ef_construction = 64);
-- Index for document lookups
CREATE INDEX idx_memory_chunks_document ON memory_chunks(document_id);
-- ==================== Heartbeat State ====================
-- Tracks periodic heartbeat execution per user/agent
CREATE TABLE heartbeat_state (
id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
user_id TEXT NOT NULL,
agent_id UUID, -- NULL = default agent for user
-- Timing
last_run TIMESTAMPTZ, -- When heartbeat last executed
next_run TIMESTAMPTZ, -- Scheduled next execution
interval_seconds INT NOT NULL DEFAULT 1800, -- 30 minutes default
-- State
enabled BOOLEAN NOT NULL DEFAULT true,
consecutive_failures INT NOT NULL DEFAULT 0,
-- Last check timestamps (for batched monitoring)
-- e.g., {"email": "2024-01-15T10:00:00Z", "calendar": "2024-01-15T10:00:00Z"}
last_checks JSONB NOT NULL DEFAULT '{}',
-- Ensure one heartbeat config per user/agent
CONSTRAINT unique_heartbeat_per_user UNIQUE (user_id, agent_id)
);
CREATE INDEX idx_heartbeat_user ON heartbeat_state(user_id);
CREATE INDEX idx_heartbeat_next_run ON heartbeat_state(next_run) WHERE enabled = true;
-- ==================== Helper Functions ====================
-- Function to update updated_at timestamp
CREATE OR REPLACE FUNCTION update_updated_at_column()
RETURNS TRIGGER AS $$
BEGIN
NEW.updated_at = NOW();
RETURN NEW;
END;
$$ language 'plpgsql';
-- Trigger to auto-update updated_at on memory_documents
CREATE TRIGGER update_memory_documents_updated_at
BEFORE UPDATE ON memory_documents
FOR EACH ROW
EXECUTE FUNCTION update_updated_at_column();
-- ==================== Views ====================
-- View for documents with chunk counts (useful for debugging)
CREATE VIEW memory_documents_summary AS
SELECT
d.id,
d.user_id,
d.doc_type,
d.title,
d.created_at,
d.updated_at,
COUNT(c.id) as chunk_count,
COUNT(c.embedding) as embedded_chunk_count
FROM memory_documents d
LEFT JOIN memory_chunks c ON c.document_id = d.id
GROUP BY d.id;
-- View for pending embedding work
CREATE VIEW chunks_pending_embedding AS
SELECT
c.id as chunk_id,
c.document_id,
d.user_id,
d.doc_type,
LENGTH(c.content) as content_length
FROM memory_chunks c
JOIN memory_documents d ON d.id = c.document_id
WHERE c.embedding IS NULL;