mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-25 14:53:34 +00:00
fix(ci): WASM WIT compat sqlite3 duplicate symbol conflict (#953)
* fix(ci): use explicit features in WASM WIT compat test to avoid sqlite3 symbol conflicts The `import` feature (added in #903) brings in `rusqlite[bundled]` which conflicts with `libsql-ffi` — both bundle SQLite C code, causing duplicate symbol linker errors. Use explicit features matching the test matrix instead of `--all-features`. Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: replace rusqlite with libsql in import module to fix sqlite3 symbol conflict The `import` feature used `rusqlite[bundled]` which bundled its own SQLite C code, conflicting with `libsql-ffi` (also bundles SQLite). This caused duplicate `sqlite3_*` symbol linker errors when both features were enabled via `--all-features`. Replace `rusqlite` with `libsql` (already a dependency) in the import reader. The `import` feature now implies `libsql`. This eliminates the duplicate symbol conflict and allows `--all-features` to compile cleanly. Also restores `--all-features` in the WASM WIT compat CI test (now safe) and converts all import test helpers from rusqlite to libsql. Co-Authored-By: Claude Opus 4.6 <[email protected]> * style: apply cargo fmt formatting fixes Co-Authored-By: Claude Opus 4.6 <[email protected]> --------- Co-authored-by: Claude Opus 4.6 <[email protected]>
This commit is contained in:
Henry Park
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
6b841bb817
commit
fe82469904
Generated
+1
-36
@@ -2787,15 +2787,6 @@ dependencies = [
|
||||
"hashbrown 0.14.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
|
||||
dependencies = [
|
||||
"hashbrown 0.14.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.5.0"
|
||||
@@ -3410,7 +3401,6 @@ dependencies = [
|
||||
"regex",
|
||||
"reqwest",
|
||||
"rig-core",
|
||||
"rusqlite",
|
||||
"rust_decimal",
|
||||
"rust_decimal_macros",
|
||||
"rustls 0.23.37",
|
||||
@@ -3707,7 +3697,7 @@ dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"fallible-iterator 0.2.0",
|
||||
"fallible-streaming-iterator",
|
||||
"hashlink 0.8.4",
|
||||
"hashlink",
|
||||
"libsql-ffi",
|
||||
"smallvec",
|
||||
]
|
||||
@@ -3770,17 +3760,6 @@ dependencies = [
|
||||
"zerocopy 0.7.35",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libsqlite3-sys"
|
||||
version = "0.30.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libyml"
|
||||
version = "0.0.5"
|
||||
@@ -5351,20 +5330,6 @@ dependencies = [
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusqlite"
|
||||
version = "0.32.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"fallible-iterator 0.3.0",
|
||||
"fallible-streaming-iterator",
|
||||
"hashlink 0.9.1",
|
||||
"libsqlite3-sys",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust_decimal"
|
||||
version = "1.40.0"
|
||||
|
||||
+1
-2
@@ -176,7 +176,6 @@ ed25519-dalek = { version = "2.2.0", features = ["std"] }
|
||||
hex = "0.4.3"
|
||||
|
||||
# OpenClaw import (feature gated)
|
||||
rusqlite = { version = "0.32", optional = true, features = ["bundled"] }
|
||||
json5 = { version = "0.4", optional = true }
|
||||
|
||||
# macOS keychain
|
||||
@@ -214,7 +213,7 @@ libsql = ["dep:libsql"]
|
||||
integration = []
|
||||
html-to-markdown = ["dep:html-to-markdown-rs", "dep:readabilityrs"]
|
||||
bedrock = ["dep:aws-config", "dep:aws-sdk-bedrockruntime", "dep:aws-smithy-types"]
|
||||
import = ["dep:rusqlite", "dep:json5"]
|
||||
import = ["dep:json5", "libsql"]
|
||||
|
||||
[[test]]
|
||||
name = "html_to_markdown"
|
||||
|
||||
@@ -77,7 +77,7 @@ impl OpenClawImporter {
|
||||
// Pre-read all conversation data to validate before writing
|
||||
let mut all_conversations = Vec::new();
|
||||
for (_agent_name, db_path) in &agent_dbs {
|
||||
match reader.read_conversations(db_path) {
|
||||
match reader.read_conversations(db_path).await {
|
||||
Ok(convs) => all_conversations.extend(convs),
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to read conversations: {}", e);
|
||||
@@ -88,7 +88,7 @@ impl OpenClawImporter {
|
||||
// Pre-read all memory chunks
|
||||
let mut all_chunks = Vec::new();
|
||||
for (_agent_name, db_path) in &agent_dbs {
|
||||
match reader.read_memory_chunks(db_path) {
|
||||
match reader.read_memory_chunks(db_path).await {
|
||||
Ok(chunks) => all_chunks.extend(chunks),
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to read memory chunks: {}", e);
|
||||
|
||||
+102
-84
@@ -80,6 +80,16 @@ pub struct OpenClawMessage {
|
||||
pub created_at: Option<chrono::DateTime<chrono::Utc>>,
|
||||
}
|
||||
|
||||
/// Open an OpenClaw SQLite database file via libsql for read-only access.
|
||||
#[cfg(feature = "import")]
|
||||
async fn open_sqlite(db_path: &Path) -> Result<libsql::Connection, ImportError> {
|
||||
let db = libsql::Builder::new_local(db_path)
|
||||
.build()
|
||||
.await
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
db.connect().map_err(|e| ImportError::Sqlite(e.to_string()))
|
||||
}
|
||||
|
||||
/// Reader for OpenClaw data files and databases.
|
||||
pub struct OpenClawReader {
|
||||
openclaw_dir: PathBuf,
|
||||
@@ -226,51 +236,52 @@ impl OpenClawReader {
|
||||
|
||||
/// Read all memory chunks from an OpenClaw SQLite database.
|
||||
#[cfg(feature = "import")]
|
||||
pub fn read_memory_chunks(
|
||||
pub async fn read_memory_chunks(
|
||||
&self,
|
||||
db_path: &Path,
|
||||
) -> Result<Vec<OpenClawMemoryChunk>, ImportError> {
|
||||
use rusqlite::Connection;
|
||||
let conn = open_sqlite(db_path).await?;
|
||||
|
||||
let conn = Connection::open(db_path).map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
|
||||
let mut stmt = conn
|
||||
.prepare("SELECT path, content, embedding, chunk_index FROM chunks")
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
|
||||
let chunks = stmt
|
||||
.query_map([], |row| {
|
||||
let path: String = row.get(0)?;
|
||||
let content: String = row.get(1)?;
|
||||
let embedding_bytes: Option<Vec<u8>> = row.get(2)?;
|
||||
let chunk_index: i32 = row.get(3)?;
|
||||
|
||||
// Convert binary embedding blob to Vec<f32> if present
|
||||
let embedding = embedding_bytes.map(|bytes| {
|
||||
bytes
|
||||
.chunks(4)
|
||||
.map(|chunk| {
|
||||
if chunk.len() == 4 {
|
||||
f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
|
||||
Ok(OpenClawMemoryChunk {
|
||||
path,
|
||||
content,
|
||||
embedding,
|
||||
chunk_index,
|
||||
})
|
||||
})
|
||||
let mut rows = conn
|
||||
.query(
|
||||
"SELECT path, content, embedding, chunk_index FROM chunks",
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
|
||||
let mut result = Vec::new();
|
||||
for chunk_result in chunks {
|
||||
result.push(chunk_result.map_err(|e| ImportError::Sqlite(e.to_string()))?);
|
||||
while let Some(row) = rows
|
||||
.next()
|
||||
.await
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?
|
||||
{
|
||||
let path: String = row.get(0).map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
let content: String = row.get(1).map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
let embedding_blob: Option<Vec<u8>> =
|
||||
row.get(2).map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
let chunk_index: i32 = row.get(3).map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
|
||||
// Convert binary embedding blob to Vec<f32> if present
|
||||
let embedding = embedding_blob.map(|bytes| {
|
||||
bytes
|
||||
.chunks(4)
|
||||
.map(|chunk| {
|
||||
if chunk.len() == 4 {
|
||||
f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]])
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
|
||||
result.push(OpenClawMemoryChunk {
|
||||
path,
|
||||
content,
|
||||
embedding,
|
||||
chunk_index,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
@@ -278,63 +289,70 @@ impl OpenClawReader {
|
||||
|
||||
/// Read all conversations from an OpenClaw SQLite database.
|
||||
#[cfg(feature = "import")]
|
||||
pub fn read_conversations(
|
||||
pub async fn read_conversations(
|
||||
&self,
|
||||
db_path: &Path,
|
||||
) -> Result<Vec<OpenClawConversation>, ImportError> {
|
||||
use rusqlite::Connection;
|
||||
let conn = open_sqlite(db_path).await?;
|
||||
|
||||
let conn = Connection::open(db_path).map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
|
||||
// First, read all conversations
|
||||
let mut conv_stmt = conn
|
||||
.prepare("SELECT id, channel, created_at FROM conversations ORDER BY created_at DESC")
|
||||
let mut conv_rows = conn
|
||||
.query(
|
||||
"SELECT id, channel, created_at FROM conversations ORDER BY created_at DESC",
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
|
||||
let mut conversations = Vec::new();
|
||||
let conv_rows = conv_stmt
|
||||
.query_map([], |row| {
|
||||
let id: String = row.get(0)?;
|
||||
let channel: String = row.get(1)?;
|
||||
let created_at: Option<String> = row.get(2)?;
|
||||
while let Some(row) = conv_rows
|
||||
.next()
|
||||
.await
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?
|
||||
{
|
||||
let id: String = row.get(0).map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
let channel: String = row.get(1).map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
let created_at: Option<String> =
|
||||
row.get(2).map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
|
||||
let created_at = created_at
|
||||
let created_at = created_at
|
||||
.and_then(|s| chrono::DateTime::parse_from_rfc3339(&s).ok())
|
||||
.map(|dt| dt.with_timezone(&chrono::Utc));
|
||||
|
||||
// Read messages for this conversation
|
||||
let mut msg_rows = conn
|
||||
.query(
|
||||
"SELECT role, content, created_at FROM messages WHERE conversation_id = ?1 ORDER BY created_at",
|
||||
libsql::params![id.as_str()],
|
||||
)
|
||||
.await
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
|
||||
let mut messages = Vec::new();
|
||||
while let Some(msg_row) = msg_rows
|
||||
.next()
|
||||
.await
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?
|
||||
{
|
||||
let role: String = msg_row
|
||||
.get(0)
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
let content: String = msg_row
|
||||
.get(1)
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
let msg_created_at: Option<String> = msg_row
|
||||
.get(2)
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
|
||||
let msg_created_at = msg_created_at
|
||||
.and_then(|s| chrono::DateTime::parse_from_rfc3339(&s).ok())
|
||||
.map(|dt| dt.with_timezone(&chrono::Utc));
|
||||
|
||||
Ok((id, channel, created_at))
|
||||
})
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
|
||||
for row_result in conv_rows {
|
||||
let (id, channel, created_at) =
|
||||
row_result.map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
|
||||
// Read messages for this conversation
|
||||
let mut msg_stmt = conn.prepare(
|
||||
"SELECT role, content, created_at FROM messages WHERE conversation_id = ? ORDER BY created_at"
|
||||
)
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
|
||||
let messages = msg_stmt
|
||||
.query_map([&id], |row| {
|
||||
let role: String = row.get(0)?;
|
||||
let content: String = row.get(1)?;
|
||||
let created_at: Option<String> = row.get(2)?;
|
||||
|
||||
let created_at = created_at
|
||||
.and_then(|s| chrono::DateTime::parse_from_rfc3339(&s).ok())
|
||||
.map(|dt| dt.with_timezone(&chrono::Utc));
|
||||
|
||||
Ok(OpenClawMessage {
|
||||
role,
|
||||
content,
|
||||
created_at,
|
||||
})
|
||||
})
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|e| ImportError::Sqlite(e.to_string()))?;
|
||||
messages.push(OpenClawMessage {
|
||||
role,
|
||||
content,
|
||||
created_at: msg_created_at,
|
||||
});
|
||||
}
|
||||
|
||||
conversations.push(OpenClawConversation {
|
||||
id,
|
||||
|
||||
@@ -47,15 +47,14 @@ mod comprehensive_import_tests {
|
||||
}
|
||||
|
||||
/// Helper to create a synthetic SQLite database with memory chunks
|
||||
fn create_synthetic_memory_db(
|
||||
async fn create_synthetic_memory_db(
|
||||
agents_dir: &Path,
|
||||
) -> Result<PathBuf, Box<dyn std::error::Error>> {
|
||||
use rusqlite::Connection;
|
||||
|
||||
std::fs::create_dir_all(agents_dir)?;
|
||||
let db_path = agents_dir.join("test_agent.sqlite");
|
||||
|
||||
let conn = Connection::open(&db_path)?;
|
||||
let db = libsql::Builder::new_local(&db_path).build().await?;
|
||||
let conn = db.connect()?;
|
||||
|
||||
// Create chunks table (simplified schema)
|
||||
conn.execute(
|
||||
@@ -66,33 +65,36 @@ mod comprehensive_import_tests {
|
||||
embedding BLOB,
|
||||
chunk_index INTEGER NOT NULL
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Insert test chunks
|
||||
conn.execute(
|
||||
"INSERT INTO chunks (id, path, content, embedding, chunk_index)
|
||||
VALUES (?, ?, ?, ?, ?)",
|
||||
rusqlite::params![
|
||||
libsql::params![
|
||||
Uuid::new_v4().to_string(),
|
||||
"test/doc.md",
|
||||
"This is test chunk 1 content.",
|
||||
None::<Vec<u8>>,
|
||||
0
|
||||
libsql::Value::Null,
|
||||
0i64
|
||||
],
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO chunks (id, path, content, embedding, chunk_index)
|
||||
VALUES (?, ?, ?, ?, ?)",
|
||||
rusqlite::params![
|
||||
libsql::params![
|
||||
Uuid::new_v4().to_string(),
|
||||
"test/doc.md",
|
||||
"This is test chunk 2 content.",
|
||||
None::<Vec<u8>>,
|
||||
1
|
||||
libsql::Value::Null,
|
||||
1i64
|
||||
],
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Create conversation table
|
||||
conn.execute(
|
||||
@@ -101,8 +103,9 @@ mod comprehensive_import_tests {
|
||||
channel TEXT NOT NULL,
|
||||
created_at TEXT
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Create messages table
|
||||
conn.execute(
|
||||
@@ -114,40 +117,44 @@ mod comprehensive_import_tests {
|
||||
created_at TEXT,
|
||||
FOREIGN KEY(conversation_id) REFERENCES conversations(id)
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Insert test conversation
|
||||
let conv_id = Uuid::new_v4().to_string();
|
||||
conn.execute(
|
||||
"INSERT INTO conversations (id, channel, created_at) VALUES (?, ?, ?)",
|
||||
rusqlite::params![&conv_id, "telegram", "2024-01-15T10:30:00Z"],
|
||||
)?;
|
||||
libsql::params![conv_id.clone(), "telegram", "2024-01-15T10:30:00Z"],
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Insert test messages
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id, conversation_id, role, content, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)",
|
||||
rusqlite::params![
|
||||
libsql::params![
|
||||
Uuid::new_v4().to_string(),
|
||||
&conv_id,
|
||||
conv_id.clone(),
|
||||
"user",
|
||||
"Hello, how are you?",
|
||||
"2024-01-15T10:30:00Z"
|
||||
],
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id, conversation_id, role, content, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)",
|
||||
rusqlite::params![
|
||||
libsql::params![
|
||||
Uuid::new_v4().to_string(),
|
||||
&conv_id,
|
||||
conv_id.clone(),
|
||||
"assistant",
|
||||
"I'm doing well, thank you for asking!",
|
||||
"2024-01-15T10:31:00Z"
|
||||
],
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok(db_path)
|
||||
}
|
||||
@@ -211,13 +218,15 @@ mod comprehensive_import_tests {
|
||||
let _ = temp_dir;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_openclaw_reader_lists_agent_dbs() {
|
||||
#[tokio::test]
|
||||
async fn test_openclaw_reader_lists_agent_dbs() {
|
||||
let (temp_dir, openclaw_path) =
|
||||
create_synthetic_openclaw_dir().expect("failed to create test data");
|
||||
|
||||
let agents_dir = openclaw_path.join("agents");
|
||||
let _db_path = create_synthetic_memory_db(&agents_dir).expect("failed to create test DB");
|
||||
let _db_path = create_synthetic_memory_db(&agents_dir)
|
||||
.await
|
||||
.expect("failed to create test DB");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("failed to create reader");
|
||||
|
||||
@@ -230,18 +239,21 @@ mod comprehensive_import_tests {
|
||||
let _ = temp_dir;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_openclaw_reader_reads_memory_chunks() {
|
||||
#[tokio::test]
|
||||
async fn test_openclaw_reader_reads_memory_chunks() {
|
||||
let (temp_dir, openclaw_path) =
|
||||
create_synthetic_openclaw_dir().expect("failed to create test data");
|
||||
|
||||
let agents_dir = openclaw_path.join("agents");
|
||||
let db_path = create_synthetic_memory_db(&agents_dir).expect("failed to create test DB");
|
||||
let db_path = create_synthetic_memory_db(&agents_dir)
|
||||
.await
|
||||
.expect("failed to create test DB");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("failed to create reader");
|
||||
|
||||
let chunks = reader
|
||||
.read_memory_chunks(&db_path)
|
||||
.await
|
||||
.expect("failed to read memory chunks");
|
||||
|
||||
// Should find 2 chunks
|
||||
@@ -260,18 +272,21 @@ mod comprehensive_import_tests {
|
||||
let _ = temp_dir;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_openclaw_reader_reads_conversations() {
|
||||
#[tokio::test]
|
||||
async fn test_openclaw_reader_reads_conversations() {
|
||||
let (temp_dir, openclaw_path) =
|
||||
create_synthetic_openclaw_dir().expect("failed to create test data");
|
||||
|
||||
let agents_dir = openclaw_path.join("agents");
|
||||
let db_path = create_synthetic_memory_db(&agents_dir).expect("failed to create test DB");
|
||||
let db_path = create_synthetic_memory_db(&agents_dir)
|
||||
.await
|
||||
.expect("failed to create test DB");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("failed to create reader");
|
||||
|
||||
let conversations = reader
|
||||
.read_conversations(&db_path)
|
||||
.await
|
||||
.expect("failed to read conversations");
|
||||
|
||||
// Should find 1 conversation
|
||||
|
||||
@@ -16,7 +16,8 @@ mod e2e_import_tests {
|
||||
use ironclaw::import::{ImportOptions, ImportStats};
|
||||
|
||||
/// Helper: Create a synthetic OpenClaw with full structure
|
||||
fn setup_full_openclaw_test_env() -> Result<(TempDir, PathBuf), Box<dyn std::error::Error>> {
|
||||
async fn setup_full_openclaw_test_env() -> Result<(TempDir, PathBuf), Box<dyn std::error::Error>>
|
||||
{
|
||||
let temp_dir = TempDir::new()?;
|
||||
let openclaw_path = temp_dir.path().to_path_buf();
|
||||
|
||||
@@ -60,17 +61,16 @@ mod e2e_import_tests {
|
||||
let agents_dir = openclaw_path.join("agents");
|
||||
std::fs::create_dir_all(&agents_dir)?;
|
||||
|
||||
create_full_agent_db(&agents_dir.join("primary_agent.sqlite"))?;
|
||||
create_full_agent_db(&agents_dir.join("secondary_agent.sqlite"))?;
|
||||
create_full_agent_db(&agents_dir.join("primary_agent.sqlite")).await?;
|
||||
create_full_agent_db(&agents_dir.join("secondary_agent.sqlite")).await?;
|
||||
|
||||
Ok((temp_dir, openclaw_path))
|
||||
}
|
||||
|
||||
/// Helper: Create a full agent SQLite database with chunks and conversations
|
||||
fn create_full_agent_db(db_path: &PathBuf) -> Result<(), Box<dyn std::error::Error>> {
|
||||
use rusqlite::Connection;
|
||||
|
||||
let conn = Connection::open(db_path)?;
|
||||
async fn create_full_agent_db(db_path: &PathBuf) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let db = libsql::Builder::new_local(db_path).build().await?;
|
||||
let conn = db.connect()?;
|
||||
|
||||
// Chunks table
|
||||
conn.execute(
|
||||
@@ -81,22 +81,24 @@ mod e2e_import_tests {
|
||||
embedding BLOB,
|
||||
chunk_index INTEGER NOT NULL
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Insert 5 chunks
|
||||
for i in 0..5 {
|
||||
conn.execute(
|
||||
"INSERT INTO chunks (id, path, content, embedding, chunk_index)
|
||||
VALUES (?, ?, ?, ?, ?)",
|
||||
rusqlite::params![
|
||||
libsql::params![
|
||||
Uuid::new_v4().to_string(),
|
||||
format!("notes/section_{}.md", i),
|
||||
format!("Content for section {}. This is important information.", i),
|
||||
None::<Vec<u8>>,
|
||||
i
|
||||
libsql::Value::Null,
|
||||
i as i64
|
||||
],
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Conversations table
|
||||
@@ -106,8 +108,9 @@ mod e2e_import_tests {
|
||||
channel TEXT NOT NULL,
|
||||
created_at TEXT
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Messages table
|
||||
conn.execute(
|
||||
@@ -119,8 +122,9 @@ mod e2e_import_tests {
|
||||
created_at TEXT,
|
||||
FOREIGN KEY(conversation_id) REFERENCES conversations(id)
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Insert 3 conversations with messages
|
||||
for conv_num in 0..3 {
|
||||
@@ -133,12 +137,13 @@ mod e2e_import_tests {
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO conversations (id, channel, created_at) VALUES (?, ?, ?)",
|
||||
rusqlite::params![
|
||||
&conv_id,
|
||||
libsql::params![
|
||||
conv_id.clone(),
|
||||
channel,
|
||||
format!("2024-01-{:02}T10:00:00Z", 10 + conv_num)
|
||||
],
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Add 3 messages per conversation
|
||||
for msg_num in 0..3 {
|
||||
@@ -150,9 +155,9 @@ mod e2e_import_tests {
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id, conversation_id, role, content, created_at)
|
||||
VALUES (?, ?, ?, ?, ?)",
|
||||
rusqlite::params![
|
||||
libsql::params![
|
||||
Uuid::new_v4().to_string(),
|
||||
&conv_id,
|
||||
conv_id.clone(),
|
||||
role,
|
||||
format!(
|
||||
"{} message {} from conversation {}",
|
||||
@@ -160,7 +165,8 @@ mod e2e_import_tests {
|
||||
),
|
||||
format!("2024-01-{:02}T10:{:02}:00Z", 10 + conv_num, msg_num * 10)
|
||||
],
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -171,9 +177,9 @@ mod e2e_import_tests {
|
||||
// Configuration & Settings Tests
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_full_config_extraction() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().expect("setup failed");
|
||||
#[tokio::test]
|
||||
async fn test_full_config_extraction() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().await.expect("setup failed");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
let config = reader.read_config().expect("config read failed");
|
||||
@@ -198,9 +204,9 @@ mod e2e_import_tests {
|
||||
assert!(config.other_settings.contains_key("custom_setting"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_settings_mapping_to_ironclaw_format() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().expect("setup failed");
|
||||
#[tokio::test]
|
||||
async fn test_settings_mapping_to_ironclaw_format() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().await.expect("setup failed");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
let config = reader.read_config().expect("config read failed");
|
||||
@@ -224,9 +230,9 @@ mod e2e_import_tests {
|
||||
// Credential Extraction Tests
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_credentials_extraction() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().expect("setup failed");
|
||||
#[tokio::test]
|
||||
async fn test_credentials_extraction() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().await.expect("setup failed");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
let config = reader.read_config().expect("config read failed");
|
||||
@@ -249,9 +255,9 @@ mod e2e_import_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_credentials_never_logged() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().expect("setup failed");
|
||||
#[tokio::test]
|
||||
async fn test_credentials_never_logged() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().await.expect("setup failed");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
let config = reader.read_config().expect("config read failed");
|
||||
@@ -271,9 +277,9 @@ mod e2e_import_tests {
|
||||
// Data Volume Tests
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_full_workspace_import_counts() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().expect("setup failed");
|
||||
#[tokio::test]
|
||||
async fn test_full_workspace_import_counts() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().await.expect("setup failed");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
|
||||
@@ -288,9 +294,9 @@ mod e2e_import_tests {
|
||||
assert_eq!(agent_dbs.len(), 2); // primary + secondary
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_full_memory_chunks_import() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().expect("setup failed");
|
||||
#[tokio::test]
|
||||
async fn test_full_memory_chunks_import() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().await.expect("setup failed");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
let agent_dbs = reader.list_agent_dbs().expect("list agent dbs failed");
|
||||
@@ -299,6 +305,7 @@ mod e2e_import_tests {
|
||||
for (_name, db_path) in agent_dbs {
|
||||
let chunks = reader
|
||||
.read_memory_chunks(&db_path)
|
||||
.await
|
||||
.expect("read memory chunks failed");
|
||||
assert_eq!(chunks.len(), 5);
|
||||
|
||||
@@ -314,9 +321,9 @@ mod e2e_import_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_full_conversations_import() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().expect("setup failed");
|
||||
#[tokio::test]
|
||||
async fn test_full_conversations_import() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().await.expect("setup failed");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
let agent_dbs = reader.list_agent_dbs().expect("list agent dbs failed");
|
||||
@@ -325,6 +332,7 @@ mod e2e_import_tests {
|
||||
for (_name, db_path) in agent_dbs {
|
||||
let conversations = reader
|
||||
.read_conversations(&db_path)
|
||||
.await
|
||||
.expect("read conversations failed");
|
||||
assert_eq!(conversations.len(), 3);
|
||||
|
||||
@@ -387,8 +395,8 @@ mod e2e_import_tests {
|
||||
// Error Handling Tests
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_error_on_corrupt_sqlite() {
|
||||
#[tokio::test]
|
||||
async fn test_error_on_corrupt_sqlite() {
|
||||
let temp_dir = TempDir::new().expect("temp dir creation failed");
|
||||
let openclaw_path = temp_dir.path().to_path_buf();
|
||||
|
||||
@@ -410,7 +418,7 @@ mod e2e_import_tests {
|
||||
assert_eq!(dbs.len(), 1);
|
||||
|
||||
// But reading should fail
|
||||
let result = reader.read_memory_chunks(&dbs[0].1);
|
||||
let result = reader.read_memory_chunks(&dbs[0].1).await;
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
@@ -437,9 +445,9 @@ mod e2e_import_tests {
|
||||
// Extensibility Tests
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_multiple_agents_independent_data() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().expect("setup failed");
|
||||
#[tokio::test]
|
||||
async fn test_multiple_agents_independent_data() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().await.expect("setup failed");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
let agent_dbs = reader.list_agent_dbs().expect("list agent dbs failed");
|
||||
@@ -453,14 +461,15 @@ mod e2e_import_tests {
|
||||
for (_name, db_path) in &agent_dbs {
|
||||
let chunks = reader
|
||||
.read_memory_chunks(db_path)
|
||||
.await
|
||||
.expect("read chunks failed");
|
||||
assert_eq!(chunks.len(), 5);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_channel_diversity_in_conversations() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().expect("setup failed");
|
||||
#[tokio::test]
|
||||
async fn test_channel_diversity_in_conversations() {
|
||||
let (_temp, openclaw_path) = setup_full_openclaw_test_env().await.expect("setup failed");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
let agent_dbs = reader.list_agent_dbs().expect("list agent dbs failed");
|
||||
@@ -468,6 +477,7 @@ mod e2e_import_tests {
|
||||
// Get conversations from first agent
|
||||
let conversations = reader
|
||||
.read_conversations(&agent_dbs[0].1)
|
||||
.await
|
||||
.expect("read conversations failed");
|
||||
|
||||
// Should have different channels
|
||||
|
||||
@@ -112,8 +112,8 @@ mod error_handling_tests {
|
||||
// SQLite Database Errors
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_error_corrupt_sqlite_file() {
|
||||
#[tokio::test]
|
||||
async fn test_error_corrupt_sqlite_file() {
|
||||
let temp_dir = TempDir::new().expect("temp dir creation failed");
|
||||
let openclaw_path = temp_dir.path().to_path_buf();
|
||||
|
||||
@@ -133,12 +133,12 @@ mod error_handling_tests {
|
||||
assert_eq!(dbs.len(), 1);
|
||||
|
||||
// But reading should fail
|
||||
let result = reader.read_memory_chunks(&dbs[0].1);
|
||||
let result = reader.read_memory_chunks(&dbs[0].1).await;
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_missing_chunks_table() {
|
||||
#[tokio::test]
|
||||
async fn test_error_missing_chunks_table() {
|
||||
let temp_dir = TempDir::new().expect("temp dir creation failed");
|
||||
let openclaw_path = temp_dir.path().to_path_buf();
|
||||
|
||||
@@ -148,14 +148,17 @@ mod error_handling_tests {
|
||||
let db_path = agents_dir.join("no_chunks.sqlite");
|
||||
|
||||
// Create valid SQLite but without chunks table
|
||||
use rusqlite::Connection;
|
||||
let conn = Connection::open(&db_path).expect("db creation failed");
|
||||
let db = libsql::Builder::new_local(&db_path)
|
||||
.build()
|
||||
.await
|
||||
.expect("db creation failed");
|
||||
let conn = db.connect().expect("connect failed");
|
||||
conn.execute(
|
||||
"CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create table failed");
|
||||
drop(conn);
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
|
||||
@@ -163,12 +166,12 @@ mod error_handling_tests {
|
||||
assert_eq!(dbs.len(), 1);
|
||||
|
||||
// Should fail: chunks table doesn't exist
|
||||
let result = reader.read_memory_chunks(&dbs[0].1);
|
||||
let result = reader.read_memory_chunks(&dbs[0].1).await;
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_missing_conversations_table() {
|
||||
#[tokio::test]
|
||||
async fn test_error_missing_conversations_table() {
|
||||
let temp_dir = TempDir::new().expect("temp dir creation failed");
|
||||
let openclaw_path = temp_dir.path().to_path_buf();
|
||||
|
||||
@@ -177,15 +180,18 @@ mod error_handling_tests {
|
||||
|
||||
let db_path = agents_dir.join("no_conversations.sqlite");
|
||||
|
||||
use rusqlite::Connection;
|
||||
let conn = Connection::open(&db_path).expect("db creation failed");
|
||||
let db = libsql::Builder::new_local(&db_path)
|
||||
.build()
|
||||
.await
|
||||
.expect("db creation failed");
|
||||
let conn = db.connect().expect("connect failed");
|
||||
// Only create chunks table, not conversations
|
||||
conn.execute(
|
||||
"CREATE TABLE chunks (id TEXT, path TEXT, content TEXT, embedding BLOB, chunk_index INTEGER)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create table failed");
|
||||
drop(conn);
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
|
||||
@@ -193,7 +199,7 @@ mod error_handling_tests {
|
||||
assert_eq!(dbs.len(), 1);
|
||||
|
||||
// Should fail: conversations table doesn't exist
|
||||
let result = reader.read_conversations(&dbs[0].1);
|
||||
let result = reader.read_conversations(&dbs[0].1).await;
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
@@ -201,8 +207,8 @@ mod error_handling_tests {
|
||||
// Edge Cases
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_edge_case_empty_chunks_table() {
|
||||
#[tokio::test]
|
||||
async fn test_edge_case_empty_chunks_table() {
|
||||
let temp_dir = TempDir::new().expect("temp dir creation failed");
|
||||
let openclaw_path = temp_dir.path().to_path_buf();
|
||||
|
||||
@@ -211,14 +217,17 @@ mod error_handling_tests {
|
||||
|
||||
let db_path = agents_dir.join("empty.sqlite");
|
||||
|
||||
use rusqlite::Connection;
|
||||
let conn = Connection::open(&db_path).expect("db creation failed");
|
||||
let db = libsql::Builder::new_local(&db_path)
|
||||
.build()
|
||||
.await
|
||||
.expect("db creation failed");
|
||||
let conn = db.connect().expect("connect failed");
|
||||
conn.execute(
|
||||
"CREATE TABLE chunks (id TEXT, path TEXT, content TEXT, embedding BLOB, chunk_index INTEGER)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create table failed");
|
||||
drop(conn);
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
|
||||
@@ -227,12 +236,13 @@ mod error_handling_tests {
|
||||
// Should succeed but return empty list
|
||||
let chunks = reader
|
||||
.read_memory_chunks(&dbs[0].1)
|
||||
.await
|
||||
.expect("read chunks failed");
|
||||
assert_eq!(chunks.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_edge_case_empty_conversations_table() {
|
||||
#[tokio::test]
|
||||
async fn test_edge_case_empty_conversations_table() {
|
||||
let temp_dir = TempDir::new().expect("temp dir creation failed");
|
||||
let openclaw_path = temp_dir.path().to_path_buf();
|
||||
|
||||
@@ -241,19 +251,23 @@ mod error_handling_tests {
|
||||
|
||||
let db_path = agents_dir.join("empty_conv.sqlite");
|
||||
|
||||
use rusqlite::Connection;
|
||||
let conn = Connection::open(&db_path).expect("db creation failed");
|
||||
let db = libsql::Builder::new_local(&db_path)
|
||||
.build()
|
||||
.await
|
||||
.expect("db creation failed");
|
||||
let conn = db.connect().expect("connect failed");
|
||||
conn.execute(
|
||||
"CREATE TABLE conversations (id TEXT, channel TEXT, created_at TEXT)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create table failed");
|
||||
conn.execute(
|
||||
"CREATE TABLE messages (id TEXT, conversation_id TEXT, role TEXT, content TEXT, created_at TEXT)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create table failed");
|
||||
drop(conn);
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
|
||||
@@ -262,12 +276,13 @@ mod error_handling_tests {
|
||||
// Should succeed but return empty list
|
||||
let conversations = reader
|
||||
.read_conversations(&dbs[0].1)
|
||||
.await
|
||||
.expect("read conversations failed");
|
||||
assert_eq!(conversations.len(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_edge_case_very_large_content() {
|
||||
#[tokio::test]
|
||||
async fn test_edge_case_very_large_content() {
|
||||
let temp_dir = TempDir::new().expect("temp dir creation failed");
|
||||
let openclaw_path = temp_dir.path().to_path_buf();
|
||||
|
||||
@@ -276,22 +291,26 @@ mod error_handling_tests {
|
||||
|
||||
let db_path = agents_dir.join("large.sqlite");
|
||||
|
||||
use rusqlite::Connection;
|
||||
let conn = Connection::open(&db_path).expect("db creation failed");
|
||||
let db = libsql::Builder::new_local(&db_path)
|
||||
.build()
|
||||
.await
|
||||
.expect("db creation failed");
|
||||
let conn = db.connect().expect("connect failed");
|
||||
conn.execute(
|
||||
"CREATE TABLE chunks (id TEXT, path TEXT, content TEXT, embedding BLOB, chunk_index INTEGER)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create table failed");
|
||||
|
||||
// Insert very large content (1MB)
|
||||
let large_content = "x".repeat(1024 * 1024);
|
||||
conn.execute(
|
||||
"INSERT INTO chunks VALUES (?, ?, ?, ?, ?)",
|
||||
rusqlite::params!["id1", "path", large_content, None::<Vec<u8>>, 0],
|
||||
libsql::params!["id1", "path", large_content, libsql::Value::Null, 0i64],
|
||||
)
|
||||
.await
|
||||
.expect("insert failed");
|
||||
drop(conn);
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
|
||||
@@ -300,13 +319,14 @@ mod error_handling_tests {
|
||||
// Should still succeed
|
||||
let chunks = reader
|
||||
.read_memory_chunks(&dbs[0].1)
|
||||
.await
|
||||
.expect("read chunks failed");
|
||||
assert_eq!(chunks.len(), 1);
|
||||
assert_eq!(chunks[0].content.len(), 1024 * 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_edge_case_special_characters_in_content() {
|
||||
#[tokio::test]
|
||||
async fn test_edge_case_special_characters_in_content() {
|
||||
let temp_dir = TempDir::new().expect("temp dir creation failed");
|
||||
let openclaw_path = temp_dir.path().to_path_buf();
|
||||
|
||||
@@ -315,22 +335,26 @@ mod error_handling_tests {
|
||||
|
||||
let db_path = agents_dir.join("special.sqlite");
|
||||
|
||||
use rusqlite::Connection;
|
||||
let conn = Connection::open(&db_path).expect("db creation failed");
|
||||
let db = libsql::Builder::new_local(&db_path)
|
||||
.build()
|
||||
.await
|
||||
.expect("db creation failed");
|
||||
let conn = db.connect().expect("connect failed");
|
||||
conn.execute(
|
||||
"CREATE TABLE chunks (id TEXT, path TEXT, content TEXT, embedding BLOB, chunk_index INTEGER)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create table failed");
|
||||
|
||||
// Insert content with special characters
|
||||
let special_content = "Content with emoji 🚀 and UTF-8: 中文, العربية, ελληνικά";
|
||||
let special_content = "Content with emoji \u{1f680} and UTF-8: \u{4e2d}\u{6587}, \u{0627}\u{0644}\u{0639}\u{0631}\u{0628}\u{064a}\u{0629}, \u{03b5}\u{03bb}\u{03bb}\u{03b7}\u{03bd}\u{03b9}\u{03ba}\u{03ac}";
|
||||
conn.execute(
|
||||
"INSERT INTO chunks VALUES (?, ?, ?, ?, ?)",
|
||||
rusqlite::params!["id1", "path", special_content, None::<Vec<u8>>, 0],
|
||||
libsql::params!["id1", "path", special_content, libsql::Value::Null, 0i64],
|
||||
)
|
||||
.await
|
||||
.expect("insert failed");
|
||||
drop(conn);
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
|
||||
@@ -339,14 +363,15 @@ mod error_handling_tests {
|
||||
// Should handle special characters
|
||||
let chunks = reader
|
||||
.read_memory_chunks(&dbs[0].1)
|
||||
.await
|
||||
.expect("read chunks failed");
|
||||
assert_eq!(chunks.len(), 1);
|
||||
assert!(chunks[0].content.contains("🚀"));
|
||||
assert!(chunks[0].content.contains("中文"));
|
||||
assert!(chunks[0].content.contains("\u{1f680}"));
|
||||
assert!(chunks[0].content.contains("\u{4e2d}\u{6587}"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_edge_case_null_values_in_fields() {
|
||||
#[tokio::test]
|
||||
async fn test_edge_case_null_values_in_fields() {
|
||||
let temp_dir = TempDir::new().expect("temp dir creation failed");
|
||||
let openclaw_path = temp_dir.path().to_path_buf();
|
||||
|
||||
@@ -355,33 +380,39 @@ mod error_handling_tests {
|
||||
|
||||
let db_path = agents_dir.join("nulls.sqlite");
|
||||
|
||||
use rusqlite::Connection;
|
||||
let conn = Connection::open(&db_path).expect("db creation failed");
|
||||
let db = libsql::Builder::new_local(&db_path)
|
||||
.build()
|
||||
.await
|
||||
.expect("db creation failed");
|
||||
let conn = db.connect().expect("connect failed");
|
||||
conn.execute(
|
||||
"CREATE TABLE conversations (id TEXT, channel TEXT, created_at TEXT)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create table failed");
|
||||
conn.execute(
|
||||
"CREATE TABLE messages (id TEXT, conversation_id TEXT, role TEXT, content TEXT, created_at TEXT)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create table failed");
|
||||
|
||||
// Insert conversation with NULL created_at
|
||||
conn.execute(
|
||||
"INSERT INTO conversations VALUES (?, ?, ?)",
|
||||
rusqlite::params!["conv1", "telegram", None::<String>],
|
||||
libsql::params!["conv1", "telegram", libsql::Value::Null],
|
||||
)
|
||||
.await
|
||||
.expect("insert failed");
|
||||
|
||||
// Insert message with NULL created_at
|
||||
conn.execute(
|
||||
"INSERT INTO messages VALUES (?, ?, ?, ?, ?)",
|
||||
rusqlite::params!["msg1", "conv1", "user", "hello", None::<String>],
|
||||
libsql::params!["msg1", "conv1", "user", "hello", libsql::Value::Null],
|
||||
)
|
||||
.await
|
||||
.expect("insert failed");
|
||||
drop(conn);
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
|
||||
@@ -390,6 +421,7 @@ mod error_handling_tests {
|
||||
// Should handle NULL timestamps gracefully
|
||||
let conversations = reader
|
||||
.read_conversations(&dbs[0].1)
|
||||
.await
|
||||
.expect("read conversations failed");
|
||||
assert_eq!(conversations.len(), 1);
|
||||
assert!(conversations[0].created_at.is_none());
|
||||
|
||||
@@ -17,7 +17,7 @@ mod idempotency_tests {
|
||||
use ironclaw::import::{ImportOptions, ImportStats};
|
||||
|
||||
/// Helper: Create minimal test OpenClaw
|
||||
fn create_minimal_openclaw() -> Result<(TempDir, PathBuf), Box<dyn std::error::Error>> {
|
||||
async fn create_minimal_openclaw() -> Result<(TempDir, PathBuf), Box<dyn std::error::Error>> {
|
||||
let temp_dir = TempDir::new()?;
|
||||
let openclaw_path = temp_dir.path().to_path_buf();
|
||||
|
||||
@@ -40,8 +40,8 @@ mod idempotency_tests {
|
||||
std::fs::create_dir_all(&agents_dir)?;
|
||||
let db_path = agents_dir.join("agent.sqlite");
|
||||
|
||||
use rusqlite::Connection;
|
||||
let conn = Connection::open(&db_path)?;
|
||||
let db = libsql::Builder::new_local(&db_path).build().await?;
|
||||
let conn = db.connect()?;
|
||||
|
||||
conn.execute(
|
||||
"CREATE TABLE chunks (
|
||||
@@ -51,24 +51,27 @@ mod idempotency_tests {
|
||||
embedding BLOB,
|
||||
chunk_index INTEGER
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO chunks VALUES (?, ?, ?, ?, ?)",
|
||||
rusqlite::params![
|
||||
libsql::params![
|
||||
Uuid::new_v4().to_string(),
|
||||
"test.md",
|
||||
"Test content",
|
||||
None::<Vec<u8>>,
|
||||
0
|
||||
libsql::Value::Null,
|
||||
0i64
|
||||
],
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
|
||||
conn.execute(
|
||||
"CREATE TABLE conversations (id TEXT PRIMARY KEY, channel TEXT, created_at TEXT)",
|
||||
[],
|
||||
)?;
|
||||
(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
conn.execute(
|
||||
"CREATE TABLE messages (
|
||||
@@ -78,8 +81,9 @@ mod idempotency_tests {
|
||||
content TEXT,
|
||||
created_at TEXT
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
Ok((temp_dir, openclaw_path))
|
||||
}
|
||||
@@ -88,9 +92,9 @@ mod idempotency_tests {
|
||||
// Idempotency Tests
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_reader_idempotent_config_reads() {
|
||||
let (_temp, openclaw_path) = create_minimal_openclaw().expect("setup failed");
|
||||
#[tokio::test]
|
||||
async fn test_reader_idempotent_config_reads() {
|
||||
let (_temp, openclaw_path) = create_minimal_openclaw().await.expect("setup failed");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
|
||||
@@ -109,9 +113,9 @@ mod idempotency_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reader_idempotent_workspace_file_listing() {
|
||||
let (_temp, openclaw_path) = create_minimal_openclaw().expect("setup failed");
|
||||
#[tokio::test]
|
||||
async fn test_reader_idempotent_workspace_file_listing() {
|
||||
let (_temp, openclaw_path) = create_minimal_openclaw().await.expect("setup failed");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
|
||||
@@ -123,9 +127,9 @@ mod idempotency_tests {
|
||||
assert_eq!(count1, 1); // MEMORY.md
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reader_idempotent_memory_chunk_reads() {
|
||||
let (_temp, openclaw_path) = create_minimal_openclaw().expect("setup failed");
|
||||
#[tokio::test]
|
||||
async fn test_reader_idempotent_memory_chunk_reads() {
|
||||
let (_temp, openclaw_path) = create_minimal_openclaw().await.expect("setup failed");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
let agent_dbs = reader.list_agent_dbs().expect("list agent dbs failed");
|
||||
@@ -134,9 +138,11 @@ mod idempotency_tests {
|
||||
// Read chunks twice
|
||||
let chunks1 = reader
|
||||
.read_memory_chunks(db_path)
|
||||
.await
|
||||
.expect("first read failed");
|
||||
let chunks2 = reader
|
||||
.read_memory_chunks(db_path)
|
||||
.await
|
||||
.expect("second read failed");
|
||||
|
||||
// Same number of chunks
|
||||
@@ -197,10 +203,10 @@ mod idempotency_tests {
|
||||
assert!(!normal_opts.dry_run);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_dry_run_stats_would_be_same() {
|
||||
#[tokio::test]
|
||||
async fn test_dry_run_stats_would_be_same() {
|
||||
// Simulating what import stats would be in dry-run vs real run
|
||||
let (_temp, openclaw_path) = create_minimal_openclaw().expect("setup failed");
|
||||
let (_temp, openclaw_path) = create_minimal_openclaw().await.expect("setup failed");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
|
||||
@@ -235,9 +241,9 @@ mod idempotency_tests {
|
||||
// Duplicate Prevention Tests
|
||||
// ────────────────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_chunk_deduplication_by_path() {
|
||||
let (_temp, openclaw_path) = create_minimal_openclaw().expect("setup failed");
|
||||
#[tokio::test]
|
||||
async fn test_chunk_deduplication_by_path() {
|
||||
let (_temp, openclaw_path) = create_minimal_openclaw().await.expect("setup failed");
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
let agent_dbs = reader.list_agent_dbs().expect("list agent dbs failed");
|
||||
@@ -245,6 +251,7 @@ mod idempotency_tests {
|
||||
|
||||
let chunks = reader
|
||||
.read_memory_chunks(db_path)
|
||||
.await
|
||||
.expect("read chunks failed");
|
||||
|
||||
// All chunks should have unique (path, chunk_index) pairs
|
||||
|
||||
@@ -4,14 +4,14 @@
|
||||
//! verifying that data is correctly stored, idempotent, and that dry-run mode
|
||||
//! prevents modifications.
|
||||
|
||||
#![cfg(all(feature = "import", feature = "libsql"))]
|
||||
#![cfg(feature = "import")]
|
||||
|
||||
#[cfg(all(feature = "import", feature = "libsql"))]
|
||||
#[cfg(feature = "import")]
|
||||
mod import_integration_tests {
|
||||
use ironclaw::db::Database;
|
||||
use ironclaw::db::libsql::LibSqlBackend;
|
||||
use ironclaw::import::ImportStats;
|
||||
use ironclaw::import::openclaw::reader::OpenClawReader;
|
||||
use ironclaw::import::{ImportOptions, ImportStats};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use tempfile::TempDir;
|
||||
@@ -29,7 +29,7 @@ mod import_integration_tests {
|
||||
}
|
||||
|
||||
/// Helper: Create a test OpenClaw directory with full structure
|
||||
fn create_test_openclaw() -> Result<(TempDir, PathBuf), Box<dyn std::error::Error>> {
|
||||
async fn create_test_openclaw() -> Result<(TempDir, PathBuf), Box<dyn std::error::Error>> {
|
||||
let temp_dir = TempDir::new()?;
|
||||
let openclaw_path = temp_dir.path().to_path_buf();
|
||||
|
||||
@@ -63,17 +63,16 @@ mod import_integration_tests {
|
||||
let agents_dir = openclaw_path.join("agents");
|
||||
std::fs::create_dir_all(&agents_dir)?;
|
||||
|
||||
create_test_agent_db(&agents_dir.join("agent1.sqlite"))?;
|
||||
create_test_agent_db(&agents_dir.join("agent2.sqlite"))?;
|
||||
create_test_agent_db(&agents_dir.join("agent1.sqlite")).await?;
|
||||
create_test_agent_db(&agents_dir.join("agent2.sqlite")).await?;
|
||||
|
||||
Ok((temp_dir, openclaw_path))
|
||||
}
|
||||
|
||||
/// Helper: Create a test agent SQLite database
|
||||
fn create_test_agent_db(db_path: &PathBuf) -> Result<(), Box<dyn std::error::Error>> {
|
||||
use rusqlite::Connection;
|
||||
|
||||
let conn = Connection::open(db_path)?;
|
||||
/// Helper: Create a test agent SQLite database using libsql
|
||||
async fn create_test_agent_db(db_path: &PathBuf) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let db = libsql::Builder::new_local(db_path).build().await?;
|
||||
let conn = db.connect()?;
|
||||
|
||||
// Chunks table
|
||||
conn.execute(
|
||||
@@ -84,27 +83,30 @@ mod import_integration_tests {
|
||||
embedding BLOB,
|
||||
chunk_index INTEGER NOT NULL
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
for i in 0..3 {
|
||||
conn.execute(
|
||||
"INSERT INTO chunks (id, path, content, embedding, chunk_index) VALUES (?, ?, ?, ?, ?)",
|
||||
rusqlite::params![
|
||||
"INSERT INTO chunks (id, path, content, embedding, chunk_index) VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
libsql::params![
|
||||
Uuid::new_v4().to_string(),
|
||||
format!("doc/section_{}.md", i),
|
||||
format!("Chunk {} content", i),
|
||||
None::<Vec<u8>>,
|
||||
i
|
||||
libsql::Value::Null,
|
||||
i as i64
|
||||
],
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
// Conversations
|
||||
conn.execute(
|
||||
"CREATE TABLE conversations (id TEXT PRIMARY KEY, channel TEXT, created_at TEXT)",
|
||||
[],
|
||||
)?;
|
||||
(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
conn.execute(
|
||||
"CREATE TABLE messages (
|
||||
@@ -114,26 +116,29 @@ mod import_integration_tests {
|
||||
content TEXT NOT NULL,
|
||||
created_at TEXT
|
||||
)",
|
||||
[],
|
||||
)?;
|
||||
(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let conv_id = Uuid::new_v4().to_string();
|
||||
conn.execute(
|
||||
"INSERT INTO conversations VALUES (?, ?, ?)",
|
||||
rusqlite::params![&conv_id, "slack", "2024-01-15T10:00:00Z"],
|
||||
)?;
|
||||
"INSERT INTO conversations VALUES (?1, ?2, ?3)",
|
||||
libsql::params![conv_id.as_str(), "slack", "2024-01-15T10:00:00Z"],
|
||||
)
|
||||
.await?;
|
||||
|
||||
for j in 0..2 {
|
||||
conn.execute(
|
||||
"INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES (?, ?, ?, ?, ?)",
|
||||
rusqlite::params![
|
||||
"INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
libsql::params![
|
||||
Uuid::new_v4().to_string(),
|
||||
&conv_id,
|
||||
conv_id.as_str(),
|
||||
if j % 2 == 0 { "user" } else { "assistant" },
|
||||
format!("Message {}", j),
|
||||
format!("2024-01-15T10:{:02}:00Z", j)
|
||||
],
|
||||
)?;
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -146,8 +151,9 @@ mod import_integration_tests {
|
||||
#[tokio::test]
|
||||
async fn test_full_import_with_database_writes() {
|
||||
let (db, _db_temp) = create_test_db().await.expect("DB creation failed");
|
||||
let (_openclaw_temp, openclaw_path) =
|
||||
create_test_openclaw().expect("OpenClaw creation failed");
|
||||
let (_openclaw_temp, openclaw_path) = create_test_openclaw()
|
||||
.await
|
||||
.expect("OpenClaw creation failed");
|
||||
|
||||
// Verify DB starts empty
|
||||
let before_docs = db
|
||||
@@ -175,12 +181,14 @@ mod import_integration_tests {
|
||||
// Read chunks from first agent
|
||||
let chunks = reader
|
||||
.read_memory_chunks(&agent_dbs[0].1)
|
||||
.await
|
||||
.expect("read chunks failed");
|
||||
assert_eq!(chunks.len(), 3); // 3 chunks created
|
||||
|
||||
// Read conversations from first agent
|
||||
let conversations = reader
|
||||
.read_conversations(&agent_dbs[0].1)
|
||||
.await
|
||||
.expect("read conversations failed");
|
||||
assert_eq!(conversations.len(), 1); // 1 conversation created
|
||||
assert_eq!(conversations[0].messages.len(), 2); // 2 messages
|
||||
@@ -192,12 +200,13 @@ mod import_integration_tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_import_command_execution() {
|
||||
let (_openclaw_temp, openclaw_path) =
|
||||
create_test_openclaw().expect("OpenClaw creation failed");
|
||||
let (_openclaw_temp, openclaw_path) = create_test_openclaw()
|
||||
.await
|
||||
.expect("OpenClaw creation failed");
|
||||
let (_db, _db_temp) = create_test_db().await.expect("DB creation failed");
|
||||
|
||||
// Create import options
|
||||
let opts = ImportOptions {
|
||||
let opts = ironclaw::import::ImportOptions {
|
||||
openclaw_path: openclaw_path.clone(),
|
||||
dry_run: false,
|
||||
re_embed: false,
|
||||
@@ -222,8 +231,9 @@ mod import_integration_tests {
|
||||
#[tokio::test]
|
||||
async fn test_dry_run_prevents_database_writes() {
|
||||
let (db, _db_temp) = create_test_db().await.expect("DB creation failed");
|
||||
let (_openclaw_temp, openclaw_path) =
|
||||
create_test_openclaw().expect("OpenClaw creation failed");
|
||||
let (_openclaw_temp, openclaw_path) = create_test_openclaw()
|
||||
.await
|
||||
.expect("OpenClaw creation failed");
|
||||
|
||||
let user_id = "test_user";
|
||||
|
||||
@@ -235,7 +245,7 @@ mod import_integration_tests {
|
||||
let before_count = before_import.len();
|
||||
|
||||
// Create import options in DRY-RUN mode
|
||||
let opts = ImportOptions {
|
||||
let opts = ironclaw::import::ImportOptions {
|
||||
openclaw_path: openclaw_path.clone(),
|
||||
dry_run: true, // ← KEY: dry_run is enabled
|
||||
re_embed: false,
|
||||
@@ -266,8 +276,9 @@ mod import_integration_tests {
|
||||
#[tokio::test]
|
||||
async fn test_import_idempotency_no_duplicates_on_reimport() {
|
||||
let (_db, _db_temp) = create_test_db().await.expect("DB creation failed");
|
||||
let (_openclaw_temp, openclaw_path) =
|
||||
create_test_openclaw().expect("OpenClaw creation failed");
|
||||
let (_openclaw_temp, openclaw_path) = create_test_openclaw()
|
||||
.await
|
||||
.expect("OpenClaw creation failed");
|
||||
|
||||
// Simulate first import: count what would be imported
|
||||
let reader1 = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
@@ -282,11 +293,13 @@ mod import_integration_tests {
|
||||
for (_, db_path) in &agent_dbs1 {
|
||||
let chunks = reader1
|
||||
.read_memory_chunks(db_path)
|
||||
.await
|
||||
.expect("read chunks failed");
|
||||
total_chunks_first += chunks.len();
|
||||
|
||||
let conversations = reader1
|
||||
.read_conversations(db_path)
|
||||
.await
|
||||
.expect("read conversations failed");
|
||||
total_conversations_first += conversations.len();
|
||||
}
|
||||
@@ -330,8 +343,9 @@ mod import_integration_tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_embedding_dimension_mismatch_queues_reembedding() {
|
||||
let (_openclaw_temp, openclaw_path) =
|
||||
create_test_openclaw().expect("OpenClaw creation failed");
|
||||
let (_openclaw_temp, openclaw_path) = create_test_openclaw()
|
||||
.await
|
||||
.expect("OpenClaw creation failed");
|
||||
|
||||
// Create an agent DB with embeddings (1536-dim)
|
||||
let agents_dir = openclaw_path.join("agents");
|
||||
@@ -339,8 +353,11 @@ mod import_integration_tests {
|
||||
let db_path = agents_dir.join("with_embeddings.sqlite");
|
||||
|
||||
{
|
||||
use rusqlite::Connection;
|
||||
let conn = Connection::open(&db_path).expect("db open failed");
|
||||
let db = libsql::Builder::new_local(&db_path)
|
||||
.build()
|
||||
.await
|
||||
.expect("db build failed");
|
||||
let conn = db.connect().expect("db connect failed");
|
||||
|
||||
conn.execute(
|
||||
"CREATE TABLE chunks (
|
||||
@@ -350,33 +367,36 @@ mod import_integration_tests {
|
||||
embedding BLOB,
|
||||
chunk_index INTEGER NOT NULL
|
||||
)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create table failed");
|
||||
|
||||
// Create a 1536-dimensional embedding (ada-002 size)
|
||||
// Each f32 is 4 bytes, so 1536 * 4 = 6144 bytes
|
||||
let embedding_1536_bytes = vec![0.1f32; 1536]
|
||||
let embedding_1536_bytes: Vec<u8> = vec![0.1f32; 1536]
|
||||
.iter()
|
||||
.flat_map(|f| f.to_le_bytes().to_vec())
|
||||
.collect::<Vec<u8>>();
|
||||
.collect();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO chunks (id, path, content, embedding, chunk_index) VALUES (?, ?, ?, ?, ?)",
|
||||
rusqlite::params![
|
||||
"INSERT INTO chunks (id, path, content, embedding, chunk_index) VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
libsql::params![
|
||||
Uuid::new_v4().to_string(),
|
||||
"test.md",
|
||||
"Chunk with embedding",
|
||||
&embedding_1536_bytes,
|
||||
0
|
||||
embedding_1536_bytes,
|
||||
0i64
|
||||
],
|
||||
)
|
||||
.await
|
||||
.expect("insert failed");
|
||||
|
||||
conn.execute(
|
||||
"CREATE TABLE conversations (id TEXT PRIMARY KEY, channel TEXT, created_at TEXT)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create conv table failed");
|
||||
|
||||
conn.execute(
|
||||
@@ -387,8 +407,9 @@ mod import_integration_tests {
|
||||
content TEXT NOT NULL,
|
||||
created_at TEXT
|
||||
)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create messages table failed");
|
||||
}
|
||||
|
||||
@@ -396,6 +417,7 @@ mod import_integration_tests {
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
let chunks = reader
|
||||
.read_memory_chunks(&db_path)
|
||||
.await
|
||||
.expect("read chunks failed");
|
||||
|
||||
assert_eq!(chunks.len(), 1);
|
||||
@@ -417,16 +439,10 @@ mod import_integration_tests {
|
||||
}
|
||||
|
||||
// Simulate dimension mismatch scenario:
|
||||
// - Source: 1536-dim (ada-002)
|
||||
// - Target: 3072-dim (text-embedding-3-large)
|
||||
// This would trigger re-embedding logic
|
||||
|
||||
let source_dim = embedding.len();
|
||||
let target_dim = 3072; // text-embedding-3-large
|
||||
|
||||
if source_dim != target_dim {
|
||||
// In real import, this would queue the chunk for re-embedding
|
||||
// Verify the logic: dimensions don't match, so chunk needs re-embedding
|
||||
assert!(
|
||||
source_dim != target_dim,
|
||||
"Dimension mismatch detected: {} -> {}",
|
||||
@@ -434,7 +450,6 @@ mod import_integration_tests {
|
||||
target_dim
|
||||
);
|
||||
|
||||
// Track that this chunk would need re-embedding
|
||||
let mut re_embed_queued = 0;
|
||||
if source_dim != target_dim {
|
||||
re_embed_queued += 1;
|
||||
@@ -466,8 +481,11 @@ mod import_integration_tests {
|
||||
let db_path = agents_dir.join("same_dim.sqlite");
|
||||
|
||||
{
|
||||
use rusqlite::Connection;
|
||||
let conn = Connection::open(&db_path).expect("db open failed");
|
||||
let db = libsql::Builder::new_local(&db_path)
|
||||
.build()
|
||||
.await
|
||||
.expect("db build failed");
|
||||
let conn = db.connect().expect("db connect failed");
|
||||
|
||||
conn.execute(
|
||||
"CREATE TABLE chunks (
|
||||
@@ -477,32 +495,35 @@ mod import_integration_tests {
|
||||
embedding BLOB,
|
||||
chunk_index INTEGER NOT NULL
|
||||
)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create table failed");
|
||||
|
||||
// 1536-dimensional embedding (text-embedding-3-small)
|
||||
let embedding_bytes = vec![0.5f32; 1536]
|
||||
let embedding_bytes: Vec<u8> = vec![0.5f32; 1536]
|
||||
.iter()
|
||||
.flat_map(|f| f.to_le_bytes().to_vec())
|
||||
.collect::<Vec<u8>>();
|
||||
.collect();
|
||||
|
||||
conn.execute(
|
||||
"INSERT INTO chunks (id, path, content, embedding, chunk_index) VALUES (?, ?, ?, ?, ?)",
|
||||
rusqlite::params![
|
||||
"INSERT INTO chunks (id, path, content, embedding, chunk_index) VALUES (?1, ?2, ?3, ?4, ?5)",
|
||||
libsql::params![
|
||||
Uuid::new_v4().to_string(),
|
||||
"test.md",
|
||||
"Chunk",
|
||||
&embedding_bytes,
|
||||
0
|
||||
embedding_bytes,
|
||||
0i64
|
||||
],
|
||||
)
|
||||
.await
|
||||
.expect("insert failed");
|
||||
|
||||
conn.execute(
|
||||
"CREATE TABLE conversations (id TEXT PRIMARY KEY, channel TEXT, created_at TEXT)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create conv table failed");
|
||||
|
||||
conn.execute(
|
||||
@@ -513,14 +534,16 @@ mod import_integration_tests {
|
||||
content TEXT NOT NULL,
|
||||
created_at TEXT
|
||||
)",
|
||||
[],
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.expect("create messages table failed");
|
||||
}
|
||||
|
||||
let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
|
||||
let chunks = reader
|
||||
.read_memory_chunks(&db_path)
|
||||
.await
|
||||
.expect("read chunks failed");
|
||||
|
||||
let embedding = chunks[0].embedding.as_ref().unwrap();
|
||||
|
||||
Reference in New Issue
Block a user