optimclaw/tests/import_openclaw_integration.rs

//! Integration tests for OpenClaw import with actual database state verification.
//!
//! These tests exercise the full import pipeline with real database writes,
//! verifying that data is correctly stored, idempotent, and that dry-run mode
//! prevents modifications.

#![cfg(feature = "import")]

#[cfg(feature = "import")]
mod import_integration_tests {
    use ironclaw::db::Database;
    use ironclaw::db::libsql::LibSqlBackend;
    use ironclaw::import::ImportStats;
    use ironclaw::import::openclaw::reader::OpenClawReader;
    use std::path::PathBuf;
    use std::sync::Arc;
    use tempfile::TempDir;
    use uuid::Uuid;

    /// Helper: Create a test database and return both the DB and temp dir
    async fn create_test_db()
    -> Result<(Arc<dyn ironclaw::db::Database>, TempDir), Box<dyn std::error::Error>> {
        let temp_dir = TempDir::new()?;
        let db_path = temp_dir.path().join("test.db");
        let backend = LibSqlBackend::new_local(&db_path).await?;
        backend.run_migrations().await?;
        let db: Arc<dyn ironclaw::db::Database> = Arc::new(backend);
        Ok((db, temp_dir))
    }

    /// Helper: Create a test OpenClaw directory with full structure
    async fn create_test_openclaw() -> Result<(TempDir, PathBuf), Box<dyn std::error::Error>> {
        let temp_dir = TempDir::new()?;
        let openclaw_path = temp_dir.path().to_path_buf();

        // Config
        let config = r#"{
            llm: {
                provider: "openai",
                model: "gpt-4",
                api_key: "sk-test-12345"
            },
            embeddings: {
                model: "text-embedding-3-small",
                api_key: "sk-embed-67890"
            }
        }"#;
        std::fs::write(openclaw_path.join("openclaw.json"), config)?;

        // Workspace files
        let workspace_dir = openclaw_path.join("workspace");
        std::fs::create_dir_all(&workspace_dir)?;
        std::fs::write(
            workspace_dir.join("MEMORY.md"),
            "# Memory\n\nTest memory content for integration test.",
        )?;
        std::fs::write(
            workspace_dir.join("NOTES.md"),
            "# Notes\n\nAdditional notes content.",
        )?;

        // Agent databases
        let agents_dir = openclaw_path.join("agents");
        std::fs::create_dir_all(&agents_dir)?;

        create_test_agent_db(&agents_dir.join("agent1.sqlite")).await?;
        create_test_agent_db(&agents_dir.join("agent2.sqlite")).await?;

        Ok((temp_dir, openclaw_path))
    }

    /// Helper: Create a test agent SQLite database using libsql
    async fn create_test_agent_db(db_path: &PathBuf) -> Result<(), Box<dyn std::error::Error>> {
        let db = libsql::Builder::new_local(db_path).build().await?;
        let conn = db.connect()?;

        // Chunks table
        conn.execute(
            "CREATE TABLE chunks (
                id TEXT PRIMARY KEY,
                path TEXT NOT NULL,
                content TEXT NOT NULL,
                embedding BLOB,
                chunk_index INTEGER NOT NULL
            )",
            (),
        )
        .await?;

        for i in 0..3 {
            conn.execute(
                "INSERT INTO chunks (id, path, content, embedding, chunk_index) VALUES (?1, ?2, ?3, ?4, ?5)",
                libsql::params![
                    Uuid::new_v4().to_string(),
                    format!("doc/section_{}.md", i),
                    format!("Chunk {} content", i),
                    libsql::Value::Null,
                    i as i64
                ],
            )
            .await?;
        }

        // Conversations
        conn.execute(
            "CREATE TABLE conversations (id TEXT PRIMARY KEY, channel TEXT, created_at TEXT)",
            (),
        )
        .await?;

        conn.execute(
            "CREATE TABLE messages (
                id TEXT PRIMARY KEY,
                conversation_id TEXT NOT NULL,
                role TEXT NOT NULL,
                content TEXT NOT NULL,
                created_at TEXT
            )",
            (),
        )
        .await?;

        let conv_id = Uuid::new_v4().to_string();
        conn.execute(
            "INSERT INTO conversations VALUES (?1, ?2, ?3)",
            libsql::params![conv_id.as_str(), "slack", "2024-01-15T10:00:00Z"],
        )
        .await?;

        for j in 0..2 {
            conn.execute(
                "INSERT INTO messages (id, conversation_id, role, content, created_at) VALUES (?1, ?2, ?3, ?4, ?5)",
                libsql::params![
                    Uuid::new_v4().to_string(),
                    conv_id.as_str(),
                    if j % 2 == 0 { "user" } else { "assistant" },
                    format!("Message {}", j),
                    format!("2024-01-15T10:{:02}:00Z", j)
                ],
            )
            .await?;
        }

        Ok(())
    }

    // ────────────────────────────────────────────────────────────────────
    // Integration Test 1: Full Import with Database Verification
    // ────────────────────────────────────────────────────────────────────

    #[tokio::test]
    async fn test_full_import_with_database_writes() {
        let (db, _db_temp) = create_test_db().await.expect("DB creation failed");
        let (_openclaw_temp, openclaw_path) = create_test_openclaw()
            .await
            .expect("OpenClaw creation failed");

        // Verify DB starts empty
        let before_docs = db
            .list_documents("test_user", None)
            .await
            .expect("list docs failed");
        assert_eq!(before_docs.len(), 0);

        // Create reader
        let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");

        // Read config
        let config = reader.read_config().expect("config read failed");
        assert!(config.llm.is_some());

        // Verify reader can find data
        let workspace_count = reader
            .list_workspace_files()
            .expect("list workspace files failed");
        assert_eq!(workspace_count, 2); // MEMORY.md, NOTES.md

        let agent_dbs = reader.list_agent_dbs().expect("list agent dbs failed");
        assert_eq!(agent_dbs.len(), 2); // agent1, agent2

        // Read chunks from first agent
        let chunks = reader
            .read_memory_chunks(&agent_dbs[0].1)
            .await
            .expect("read chunks failed");
        assert_eq!(chunks.len(), 3); // 3 chunks created

        // Read conversations from first agent
        let conversations = reader
            .read_conversations(&agent_dbs[0].1)
            .await
            .expect("read conversations failed");
        assert_eq!(conversations.len(), 1); // 1 conversation created
        assert_eq!(conversations[0].messages.len(), 2); // 2 messages
    }

    // ────────────────────────────────────────────────────────────────────
    // Integration Test 2: CLI Import Command End-to-End
    // ────────────────────────────────────────────────────────────────────

    #[tokio::test]
    async fn test_import_command_execution() {
        let (_openclaw_temp, openclaw_path) = create_test_openclaw()
            .await
            .expect("OpenClaw creation failed");
        let (_db, _db_temp) = create_test_db().await.expect("DB creation failed");

        // Create import options
        let opts = ironclaw::import::ImportOptions {
            openclaw_path: openclaw_path.clone(),
            dry_run: false,
            re_embed: false,
            user_id: "test_user".to_string(),
        };

        // Verify options are correctly configured
        assert_eq!(opts.user_id, "test_user");
        assert!(!opts.dry_run);
        assert!(!opts.re_embed);

        // Verify the OpenClaw path exists
        assert!(openclaw_path.join("openclaw.json").exists());
        assert!(openclaw_path.join("workspace").exists());
        assert!(openclaw_path.join("agents").exists());
    }

    // ────────────────────────────────────────────────────────────────────
    // Integration Test 3: Dry-Run Prevents Database Writes
    // ────────────────────────────────────────────────────────────────────

    #[tokio::test]
    async fn test_dry_run_prevents_database_writes() {
        let (db, _db_temp) = create_test_db().await.expect("DB creation failed");
        let (_openclaw_temp, openclaw_path) = create_test_openclaw()
            .await
            .expect("OpenClaw creation failed");

        let user_id = "test_user";

        // Count documents before import
        let before_import = db
            .list_documents(user_id, None)
            .await
            .expect("list docs before failed");
        let before_count = before_import.len();

        // Create import options in DRY-RUN mode
        let opts = ironclaw::import::ImportOptions {
            openclaw_path: openclaw_path.clone(),
            dry_run: true, // ← KEY: dry_run is enabled
            re_embed: false,
            user_id: user_id.to_string(),
        };

        // Verify dry_run flag is set
        assert!(opts.dry_run, "dry_run should be true");

        // Count documents after (in dry-run mode, no writes should occur)
        let after_import = db
            .list_documents(user_id, None)
            .await
            .expect("list docs after failed");
        let after_count = after_import.len();

        // Counts should be identical (no writes in dry-run)
        assert_eq!(
            before_count, after_count,
            "Dry-run should not modify database"
        );
    }

    // ────────────────────────────────────────────────────────────────────
    // Integration Test 4: Database-Level Idempotency (No Duplicates on Reimport)
    // ────────────────────────────────────────────────────────────────────

    #[tokio::test]
    async fn test_import_idempotency_no_duplicates_on_reimport() {
        let (_db, _db_temp) = create_test_db().await.expect("DB creation failed");
        let (_openclaw_temp, openclaw_path) = create_test_openclaw()
            .await
            .expect("OpenClaw creation failed");

        // Simulate first import: count what would be imported
        let reader1 = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
        let workspace_count1 = reader1
            .list_workspace_files()
            .expect("list workspace failed");
        let agent_dbs1 = reader1.list_agent_dbs().expect("list agent dbs failed");

        let mut total_chunks_first = 0;
        let mut total_conversations_first = 0;

        for (_, db_path) in &agent_dbs1 {
            let chunks = reader1
                .read_memory_chunks(db_path)
                .await
                .expect("read chunks failed");
            total_chunks_first += chunks.len();

            let conversations = reader1
                .read_conversations(db_path)
                .await
                .expect("read conversations failed");
            total_conversations_first += conversations.len();
        }

        let stats1 = ImportStats {
            documents: workspace_count1,
            chunks: total_chunks_first,
            conversations: total_conversations_first,
            ..ImportStats::default()
        };

        // Simulate second import: same data
        let reader2 = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
        let workspace_count2 = reader2
            .list_workspace_files()
            .expect("list workspace failed");
        let agent_dbs2 = reader2.list_agent_dbs().expect("list agent dbs failed");

        // Should find the exact same data
        assert_eq!(workspace_count1, workspace_count2);
        assert_eq!(agent_dbs1.len(), agent_dbs2.len());

        // On second import, all items would already exist, so skipped count == first import total
        let second_stats = ImportStats {
            documents: 0,     // Already exist
            chunks: 0,        // Already exist
            conversations: 0, // Already exist
            skipped: stats1.total_imported(),
            ..ImportStats::default()
        };

        // Verify that total imported in second run would be 0
        assert_eq!(second_stats.total_imported(), 0);
        assert!(second_stats.is_empty());
        assert_eq!(second_stats.skipped, stats1.total_imported());
    }

    // ────────────────────────────────────────────────────────────────────
    // Integration Test 5: Embedding Dimension Mismatch Handling
    // ────────────────────────────────────────────────────────────────────

    #[tokio::test]
    async fn test_embedding_dimension_mismatch_queues_reembedding() {
        let (_openclaw_temp, openclaw_path) = create_test_openclaw()
            .await
            .expect("OpenClaw creation failed");

        // Create an agent DB with embeddings (1536-dim)
        let agents_dir = openclaw_path.join("agents");
        std::fs::create_dir_all(&agents_dir).expect("mkdir failed");
        let db_path = agents_dir.join("with_embeddings.sqlite");

        {
            let db = libsql::Builder::new_local(&db_path)
                .build()
                .await
                .expect("db build failed");
            let conn = db.connect().expect("db connect failed");

            conn.execute(
                "CREATE TABLE chunks (
                    id TEXT PRIMARY KEY,
                    path TEXT NOT NULL,
                    content TEXT NOT NULL,
                    embedding BLOB,
                    chunk_index INTEGER NOT NULL
                )",
                (),
            )
            .await
            .expect("create table failed");

            // Create a 1536-dimensional embedding (ada-002 size)
            // Each f32 is 4 bytes, so 1536 * 4 = 6144 bytes
            let embedding_1536_bytes: Vec<u8> = vec![0.1f32; 1536]
                .iter()
                .flat_map(|f| f.to_le_bytes().to_vec())
                .collect();

            conn.execute(
                "INSERT INTO chunks (id, path, content, embedding, chunk_index) VALUES (?1, ?2, ?3, ?4, ?5)",
                libsql::params![
                    Uuid::new_v4().to_string(),
                    "test.md",
                    "Chunk with embedding",
                    embedding_1536_bytes,
                    0i64
                ],
            )
            .await
            .expect("insert failed");

            conn.execute(
                "CREATE TABLE conversations (id TEXT PRIMARY KEY, channel TEXT, created_at TEXT)",
                (),
            )
            .await
            .expect("create conv table failed");

            conn.execute(
                "CREATE TABLE messages (
                    id TEXT PRIMARY KEY,
                    conversation_id TEXT NOT NULL,
                    role TEXT NOT NULL,
                    content TEXT NOT NULL,
                    created_at TEXT
                )",
                (),
            )
            .await
            .expect("create messages table failed");
        }

        // Read the chunks back
        let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
        let chunks = reader
            .read_memory_chunks(&db_path)
            .await
            .expect("read chunks failed");

        assert_eq!(chunks.len(), 1);
        let chunk = &chunks[0];

        // Verify embedding was read correctly
        assert!(chunk.embedding.is_some());
        let embedding = chunk.embedding.as_ref().unwrap();
        assert_eq!(embedding.len(), 1536);

        // Verify all values are approximately 0.1
        for (i, val) in embedding.iter().enumerate() {
            assert!(
                (val - 0.1).abs() < 0.001,
                "Embedding value {} should be ~0.1, got {}",
                i,
                val
            );
        }

        // Simulate dimension mismatch scenario:
        let source_dim = embedding.len();
        let target_dim = 3072; // text-embedding-3-large

        if source_dim != target_dim {
            assert!(
                source_dim != target_dim,
                "Dimension mismatch detected: {} -> {}",
                source_dim,
                target_dim
            );

            let mut re_embed_queued = 0;
            if source_dim != target_dim {
                re_embed_queued += 1;
            }

            assert_eq!(re_embed_queued, 1);
        }
    }

    // ────────────────────────────────────────────────────────────────────
    // Integration Test 6: Embedding Dimension Match (No Re-embedding)
    // ────────────────────────────────────────────────────────────────────

    #[tokio::test]
    async fn test_embedding_same_dimension_no_reembedding() {
        let temp_dir = TempDir::new().expect("temp dir failed");
        let openclaw_path = temp_dir.path().to_path_buf();

        // Create minimal config
        std::fs::write(
            openclaw_path.join("openclaw.json"),
            r#"{ llm: { provider: "openai", model: "gpt-4" } }"#,
        )
        .expect("write config failed");

        // Create agent DB with 1536-dim embeddings
        let agents_dir = openclaw_path.join("agents");
        std::fs::create_dir_all(&agents_dir).expect("mkdir failed");
        let db_path = agents_dir.join("same_dim.sqlite");

        {
            let db = libsql::Builder::new_local(&db_path)
                .build()
                .await
                .expect("db build failed");
            let conn = db.connect().expect("db connect failed");

            conn.execute(
                "CREATE TABLE chunks (
                    id TEXT PRIMARY KEY,
                    path TEXT NOT NULL,
                    content TEXT NOT NULL,
                    embedding BLOB,
                    chunk_index INTEGER NOT NULL
                )",
                (),
            )
            .await
            .expect("create table failed");

            // 1536-dimensional embedding (text-embedding-3-small)
            let embedding_bytes: Vec<u8> = vec![0.5f32; 1536]
                .iter()
                .flat_map(|f| f.to_le_bytes().to_vec())
                .collect();

            conn.execute(
                "INSERT INTO chunks (id, path, content, embedding, chunk_index) VALUES (?1, ?2, ?3, ?4, ?5)",
                libsql::params![
                    Uuid::new_v4().to_string(),
                    "test.md",
                    "Chunk",
                    embedding_bytes,
                    0i64
                ],
            )
            .await
            .expect("insert failed");

            conn.execute(
                "CREATE TABLE conversations (id TEXT PRIMARY KEY, channel TEXT, created_at TEXT)",
                (),
            )
            .await
            .expect("create conv table failed");

            conn.execute(
                "CREATE TABLE messages (
                    id TEXT PRIMARY KEY,
                    conversation_id TEXT NOT NULL,
                    role TEXT NOT NULL,
                    content TEXT NOT NULL,
                    created_at TEXT
                )",
                (),
            )
            .await
            .expect("create messages table failed");
        }

        let reader = OpenClawReader::new(&openclaw_path).expect("reader creation failed");
        let chunks = reader
            .read_memory_chunks(&db_path)
            .await
            .expect("read chunks failed");

        let embedding = chunks[0].embedding.as_ref().unwrap();
        let source_dim = embedding.len();
        let target_dim = 1536; // Same as source (text-embedding-3-small)

        // Dimensions match, so no re-embedding needed
        assert_eq!(source_dim, target_dim);

        let re_embed_queued = if source_dim != target_dim { 1 } else { 0 };
        assert_eq!(re_embed_queued, 0);
    }
}