From 327e009622d84b911db9423d5ac40c94890f61c2 Mon Sep 17 00:00:00 2001 From: ILGIN KANAT Date: Wed, 18 Feb 2026 11:06:09 +0400 Subject: [PATCH] feat: add LanceDB support for workspace semantic search - Introduced optional LanceDB vector store for semantic search, configurable via environment variables. - Updated `.env.example` and `Cargo.toml` to include LanceDB settings. - Enhanced `DatabaseConfig` to support vector backend selection and LanceDB path configuration. - Implemented `VectorBackend` enum to manage vector store options. - Added functionality to connect to LanceDB in the database connection logic. - Updated relevant documentation to reflect new features and configuration options. This change allows users to leverage LanceDB as an alternative to pgvector/libsql for improved search capabilities. --- .env.example | 6 + CLAUDE.md | 6 + Cargo.lock | 2977 +++++++++++++++++++++++++++++++- Cargo.toml | 4 + FEATURE_PARITY.md | 2 +- src/config.rs | 81 + src/db/lancedb_wrapper.rs | 664 +++++++ src/db/libsql_backend.rs | 40 + src/db/mod.rs | 38 +- src/db/postgres.rs | 7 + src/main.rs | 38 +- src/workspace/lancedb_store.rs | 623 +++++++ src/workspace/mod.rs | 4 + src/workspace/repository.rs | 30 + tests/lancedb_integration.rs | 159 ++ 15 files changed, 4661 insertions(+), 18 deletions(-) create mode 100644 src/db/lancedb_wrapper.rs create mode 100644 src/workspace/lancedb_store.rs create mode 100644 tests/lancedb_integration.rs diff --git a/.env.example b/.env.example index 7ff45444..ddadd234 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,12 @@ DATABASE_URL=postgres://localhost/ironclaw DATABASE_POOL_SIZE=10 +# Vector store for workspace memory (optional) +# When set to "lancedb", uses LanceDB for semantic search instead of pgvector/libsql +# VECTOR_BACKEND=pgvector # default: use database's built-in (pgvector or libsql_vector_idx) +# VECTOR_BACKEND=lancedb +# LANCEDB_PATH=~/.ironclaw/lancedb # path for LanceDB when VECTOR_BACKEND=lancedb + # LLM Provider (NEAR AI) # NEAR AI provides a unified interface to all models with user authentication # Session token is stored in ~/.ironclaw/session.json and managed automatically. diff --git a/CLAUDE.md b/CLAUDE.md index ff89c399..ff15ebbf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -401,6 +401,12 @@ LIBSQL_PATH=~/.ironclaw/ironclaw.db # Default path # libSQL (Turso cloud sync) LIBSQL_URL=libsql://your-db.turso.io LIBSQL_AUTH_TOKEN=your-token # Required when LIBSQL_URL is set + +# Vector store for workspace semantic search (optional) +# When lancedb: uses LanceDB instead of pgvector/libsql for vector search +VECTOR_BACKEND=builtin # default: use database built-in +# VECTOR_BACKEND=lancedb # requires: cargo build --features lancedb +# LANCEDB_PATH=~/.ironclaw/lancedb # default when VECTOR_BACKEND=lancedb ``` ### Current Limitations (libSQL backend) diff --git a/Cargo.lock b/Cargo.lock index f529c910..eef9716c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "gimli", ] +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aead" version = "0.5.2" @@ -64,6 +70,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", + "const-random", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy 0.8.37", @@ -78,6 +86,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -155,6 +178,15 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +[[package]] +name = "approx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" +dependencies = [ + "num-traits", +] + [[package]] name = "ar_archive_writer" version = "0.5.1" @@ -170,6 +202,15 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +[[package]] +name = "arc-swap" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f3647c145568cec02c42054e07bdf9a5a698e15b466fb2341bfc393cd24aa5" +dependencies = [ + "rustversion", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -182,6 +223,230 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "arrow" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4754a624e5ae42081f464514be454b39711daae0458906dacde5f4c632f33a8" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-csv", + "arrow-data", + "arrow-ipc", + "arrow-json", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b3141e0ec5145a22d8694ea8b6d6f69305971c4fa1c1a13ef0195aef2d678b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num-traits", +] + +[[package]] +name = "arrow-array" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" +dependencies = [ + "ahash 0.8.12", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "chrono-tz", + "half", + "hashbrown 0.16.1", + "num-complex", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-buffer" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c697ddca96183182f35b3a18e50b9110b11e916d7b7799cbfd4d34662f2c56c2" +dependencies = [ + "bytes", + "half", + "num-bigint", + "num-traits", +] + +[[package]] +name = "arrow-cast" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "646bbb821e86fd57189c10b4fcdaa941deaf4181924917b0daa92735baa6ada5" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "atoi", + "base64 0.22.1", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num-traits", + "ryu", +] + +[[package]] +name = "arrow-csv" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da746f4180004e3ce7b83c977daf6394d768332349d3d913998b10a120b790a" +dependencies = [ + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "csv", + "csv-core", + "regex", +] + +[[package]] +name = "arrow-data" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fdd994a9d28e6365aa78e15da3f3950c0fdcea6b963a12fa1c391afb637b304" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num-integer", + "num-traits", +] + +[[package]] +name = "arrow-ipc" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "abf7df950701ab528bf7c0cf7eeadc0445d03ef5d6ffc151eaae6b38a58feff1" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "flatbuffers", + "lz4_flex 0.12.0", + "zstd", +] + +[[package]] +name = "arrow-json" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ff8357658bedc49792b13e2e862b80df908171275f8e6e075c460da5ee4bf86" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "indexmap 2.13.0", + "itoa", + "lexical-core", + "memchr", + "num-traits", + "ryu", + "serde_core", + "serde_json", + "simdutf8", +] + +[[package]] +name = "arrow-ord" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d8f1870e03d4cbed632959498bcc84083b5a24bded52905ae1695bd29da45b" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18228633bad92bff92a95746bbeb16e5fc318e8382b75619dec26db79e4de4c0" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" +dependencies = [ + "bitflags 2.10.0", + "serde_core", + "serde_json", +] + +[[package]] +name = "arrow-select" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" +dependencies = [ + "ahash 0.8.12", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num-traits", +] + +[[package]] +name = "arrow-string" +version = "57.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e968097061b3c0e9fe3079cf2e703e487890700546b5b0647f60fca1b5a8d8" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num-traits", + "regex", + "regex-syntax", +] + [[package]] name = "as-any" version = "0.3.2" @@ -212,6 +477,18 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "async-compression" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68650b7df54f0293fd061972a0fb05aaf4fc0879d3b3d21a638a182c5c543b9f" +dependencies = [ + "compression-codecs", + "compression-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "async-executor" version = "1.13.3" @@ -352,6 +629,24 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "async_cell" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447ab28afbb345f5408b120702a44e5529ebf90b1796ec76e9528df8e288e6c2" +dependencies = [ + "loom", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" @@ -476,6 +771,19 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bincode" version = "1.3.3" @@ -520,6 +828,15 @@ version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +[[package]] +name = "bitpacking" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a7139abd3d9cebf8cd6f920a389cf3dc9576172e32f4563f188cae3c3eb019" +dependencies = [ + "crunchy", +] + [[package]] name = "bitvec" version = "1.0.1" @@ -532,6 +849,15 @@ dependencies = [ "wyz", ] +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + [[package]] name = "blake3" version = "1.8.3" @@ -627,6 +953,31 @@ dependencies = [ "serde_with", ] +[[package]] +name = "bon" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d13a61f2963b88eef9c1be03df65d42f6996dfeac1054870d950fcf66686f83" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d314cc62af2b6b0c65780555abb4d02a03dd3b799cd42419044f0c38d99738c0" +dependencies = [ + "darling", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.114", +] + [[package]] name = "borsh" version = "1.6.0" @@ -650,6 +1001,27 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "brotli" +version = "8.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "874bb8112abecc98cbd6d81ea4fa7e94fb9449648c93cc89aa40c81c24d7de03" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.19.1" @@ -681,6 +1053,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + [[package]] name = "byteorder" version = "1.5.0" @@ -795,13 +1173,19 @@ dependencies = [ "shlex", ] +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + [[package]] name = "cexpr" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" dependencies = [ - "nom", + "nom 7.1.3", ] [[package]] @@ -830,6 +1214,16 @@ dependencies = [ "windows-link", ] +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + [[package]] name = "cipher" version = "0.4.4" @@ -915,6 +1309,33 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "comfy-table" +version = "7.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "958c5d6ecf1f214b4c2bbbbf6ab9523a864bd136dcf71a7e8904799acfe1ad47" +dependencies = [ + "unicode-segmentation", + "unicode-width 0.2.0", +] + +[[package]] +name = "compression-codecs" +version = "0.4.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00828ba6fd27b45a448e57dbfe84f1029d4c9f26b368157e9a448a5f49a2ec2a" +dependencies = [ + "compression-core", + "flate2", + "memchr", +] + +[[package]] +name = "compression-core" +version = "0.4.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75984efb6ed102a0d42db99afb6c1948f0380d1d91808d5529916e6c08b49d8d" + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -924,6 +1345,26 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -1141,7 +1582,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eee8b2b4516038bc0f1d3c9934bcb4a13dd316e04abbc63c96757a6d75978532" dependencies = [ "chrono", - "nom", + "nom 7.1.3", "once_cell", ] @@ -1244,6 +1685,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -1255,6 +1702,27 @@ dependencies = [ "typenum", ] +[[package]] +name = "csv" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52cd9d68cf7efc6ddfaaee42e7288d3a99d613d4b50f76ce9827ae0c6e14f938" +dependencies = [ + "csv-core", + "itoa", + "ryu", + "serde_core", +] + +[[package]] +name = "csv-core" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" +dependencies = [ + "memchr", +] + [[package]] name = "ctr" version = "0.9.2" @@ -1299,12 +1767,626 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "dashmap" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" +[[package]] +name = "datafusion" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ba7cb113e9c0bedf9e9765926031e132fa05a1b09ba6e93a6d1a4d7044457b8" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "bytes", + "chrono", + "datafusion-catalog", + "datafusion-catalog-listing", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-datasource-arrow", + "datafusion-datasource-csv", + "datafusion-datasource-json", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", + "datafusion-functions-table", + "datafusion-functions-window", + "datafusion-optimizer", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-optimizer", + "datafusion-physical-plan", + "datafusion-session", + "datafusion-sql", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "rand 0.9.2", + "regex", + "rstest", + "sqlparser", + "tempfile", + "tokio", + "url", + "uuid", +] + +[[package]] +name = "datafusion-catalog" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66a3a799f914a59b1ea343906a0486f17061f39509af74e874a866428951130d" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "parking_lot", + "tokio", +] + +[[package]] +name = "datafusion-catalog-listing" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db1b113c80d7a0febcd901476a57aef378e717c54517a163ed51417d87621b0" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "futures", + "itertools 0.14.0", + "log", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-common" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c10f7659e96127d25e8366be7c8be4109595d6a2c3eac70421f380a7006a1b0" +dependencies = [ + "ahash 0.8.12", + "arrow", + "arrow-ipc", + "chrono", + "half", + "hashbrown 0.14.5", + "indexmap 2.13.0", + "libc", + "log", + "object_store", + "paste", + "sqlparser", + "tokio", + "web-time", +] + +[[package]] +name = "datafusion-common-runtime" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b92065bbc6532c6651e2f7dd30b55cba0c7a14f860c7e1d15f165c41a1868d95" +dependencies = [ + "futures", + "log", + "tokio", +] + +[[package]] +name = "datafusion-datasource" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fde13794244bc7581cd82f6fff217068ed79cdc344cafe4ab2c3a1c3510b38d6" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-expr-adapter", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "glob", + "itertools 0.14.0", + "log", + "object_store", + "rand 0.9.2", + "tokio", + "url", +] + +[[package]] +name = "datafusion-datasource-arrow" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "804fa9b4ecf3157982021770617200ef7c1b2979d57bec9044748314775a9aea" +dependencies = [ + "arrow", + "arrow-ipc", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "itertools 0.14.0", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-datasource-csv" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61a1641a40b259bab38131c5e6f48fac0717bedb7dc93690e604142a849e0568" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "regex", + "tokio", +] + +[[package]] +name = "datafusion-datasource-json" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adeacdb00c1d37271176f8fb6a1d8ce096baba16ea7a4b2671840c5c9c64fe85" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-datasource", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-session", + "futures", + "object_store", + "tokio", +] + +[[package]] +name = "datafusion-doc" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b99e13947667b36ad713549237362afb054b2d8f8cc447751e23ec61202db07" + +[[package]] +name = "datafusion-execution" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63695643190679037bc946ad46a263b62016931547bf119859c511f7ff2f5178" +dependencies = [ + "arrow", + "async-trait", + "dashmap", + "datafusion-common", + "datafusion-expr", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.2", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-expr" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a4787cbf5feb1ab351f789063398f67654a6df75c4d37d7f637dc96f951a91" +dependencies = [ + "arrow", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", + "indexmap 2.13.0", + "itertools 0.14.0", + "paste", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr-common" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ce2fb1b8c15c9ac45b0863c30b268c69dc9ee7a1ee13ecf5d067738338173dc" +dependencies = [ + "arrow", + "datafusion-common", + "indexmap 2.13.0", + "itertools 0.14.0", + "paste", +] + +[[package]] +name = "datafusion-functions" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "794a9db7f7b96b3346fc007ff25e994f09b8f0511b4cf7dff651fadfe3ebb28f" +dependencies = [ + "arrow", + "arrow-buffer", + "base64 0.22.1", + "blake2", + "blake3", + "chrono", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", + "hex", + "itertools 0.14.0", + "log", + "md-5", + "num-traits", + "rand 0.9.2", + "regex", + "sha2", + "unicode-segmentation", + "uuid", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c25210520a9dcf9c2b2cbbce31ebd4131ef5af7fc60ee92b266dc7d159cb305" +dependencies = [ + "ahash 0.8.12", + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "half", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62f4a66f3b87300bb70f4124b55434d2ae3fe80455f3574701d0348da040b55d" +dependencies = [ + "ahash 0.8.12", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-functions-nested" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae5c06eed03918dc7fe7a9f082a284050f0e9ecf95d72f57712d1496da03b8c4" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr-common", + "itertools 0.14.0", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-table" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db4fed1d71738fbe22e2712d71396db04c25de4111f1ec252b8f4c6d3b25d7f5" +dependencies = [ + "arrow", + "async-trait", + "datafusion-catalog", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", + "paste", +] + +[[package]] +name = "datafusion-functions-window" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d92206aa5ae21892f1552b4d61758a862a70956e6fd7a95cb85db1de74bc6d1" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "log", + "paste", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53ae9bcc39800820d53a22d758b3b8726ff84a5a3e24cecef04ef4e5fdf1c7cc" +dependencies = [ + "datafusion-common", + "datafusion-physical-expr-common", +] + +[[package]] +name = "datafusion-macros" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1063ad4c9e094b3f798acee16d9a47bd7372d9699be2de21b05c3bd3f34ab848" +dependencies = [ + "datafusion-doc", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "datafusion-optimizer" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f35f9ec5d08b87fd1893a30c2929f2559c2f9806ca072d8fefca5009dc0f06a" +dependencies = [ + "arrow", + "chrono", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "indexmap 2.13.0", + "itertools 0.14.0", + "log", + "regex", + "regex-syntax", +] + +[[package]] +name = "datafusion-physical-expr" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c30cc8012e9eedcb48bbe112c6eff4ae5ed19cf3003cb0f505662e88b7014c5d" +dependencies = [ + "ahash 0.8.12", + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", + "half", + "hashbrown 0.14.5", + "indexmap 2.13.0", + "itertools 0.14.0", + "parking_lot", + "paste", + "petgraph", +] + +[[package]] +name = "datafusion-physical-expr-adapter" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f9ff2dbd476221b1f67337699eff432781c4e6e1713d2aefdaa517dfbf79768" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90da43e1ec550b172f34c87ec68161986ced70fd05c8d2a2add66eef9c276f03" +dependencies = [ + "ahash 0.8.12", + "arrow", + "datafusion-common", + "datafusion-expr-common", + "hashbrown 0.14.5", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-optimizer" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce9804f799acd7daef3be7aaffe77c0033768ed8fdbf5fb82fc4c5f2e6bc14e6" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "datafusion-pruning", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-physical-plan" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0acf0ad6b6924c6b1aa7d213b181e012e2d3ec0a64ff5b10ee6282ab0f8532ac" +dependencies = [ + "ahash 0.8.12", + "arrow", + "arrow-ord", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common", + "datafusion-common-runtime", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "futures", + "half", + "hashbrown 0.14.5", + "indexmap 2.13.0", + "itertools 0.14.0", + "log", + "parking_lot", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "datafusion-pruning" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac2c2498a1f134a9e11a9f5ed202a2a7d7e9774bd9249295593053ea3be999db" +dependencies = [ + "arrow", + "datafusion-common", + "datafusion-datasource", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", + "datafusion-physical-plan", + "itertools 0.14.0", + "log", +] + +[[package]] +name = "datafusion-session" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f96eebd17555386f459037c65ab73aae8df09f464524c709d6a3134ad4f4776" +dependencies = [ + "async-trait", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-plan", + "parking_lot", +] + +[[package]] +name = "datafusion-sql" +version = "51.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fc195fe60634b2c6ccfd131b487de46dc30eccae8a3c35a13f136e7f440414f" +dependencies = [ + "arrow", + "bigdecimal", + "chrono", + "datafusion-common", + "datafusion-expr", + "indexmap 2.13.0", + "log", + "regex", + "sqlparser", +] + [[package]] name = "deadpool" version = "0.12.3" @@ -1349,6 +2431,26 @@ dependencies = [ "uuid", ] +[[package]] +name = "deepsize" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cdb987ec36f6bf7bfbea3f928b75590b736fc42af8e54d97592481351b2b96c" +dependencies = [ + "deepsize_derive", +] + +[[package]] +name = "deepsize_derive" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990101d41f3bc8c1a45641024377ee284ecc338e5ecf3ea0f0e236d897c72796" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "deranged" version = "0.5.5" @@ -1497,12 +2599,28 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "downcast-rs" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "117240f60069e65410b3ae1bb213295bd828f707b5bec6596a1afc8793ce0cbc" + [[package]] name = "dyn-clone" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" +[[package]] +name = "earcutr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79127ed59a85d7687c409e9978547cffb7dc79675355ed22da6b66fd5f6ead01" +dependencies = [ + "itertools 0.11.0", + "num-traits", +] + [[package]] name = "either" version = "1.15.0" @@ -1596,6 +2714,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "ethnum" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca81e6b4777c89fd810c25a4be2b1bd93ea034fbe58e6a75216a34c6b82c539b" + [[package]] name = "event-listener" version = "5.4.1" @@ -1624,7 +2748,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74fef4569247a5f429d9156b9d0a2599914385dd189c539334c625d8099d90ab" dependencies = [ "futures-core", - "nom", + "nom 7.1.3", "pin-project-lite", ] @@ -1646,6 +2770,18 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" +[[package]] +name = "fast-float2" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb564c5c7423d25c886fb561d1e4ee69f72354d16918afa32c08811f6b6a55" + +[[package]] +name = "fastdivide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" + [[package]] name = "fastrand" version = "2.3.0" @@ -1680,6 +2816,38 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fixedbitset" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99" + +[[package]] +name = "flatbuffers" +version = "25.12.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" +dependencies = [ + "bitflags 2.10.0", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "float_next_after" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8" + [[package]] name = "fnv" version = "1.0.7" @@ -1737,6 +2905,35 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "fs4" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" +dependencies = [ + "rustix 0.38.44", + "windows-sys 0.52.0", +] + +[[package]] +name = "fsst" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f03a771ab914e207dd26bd2f12666839555ec8ecc7e1770e1ed6f9900d899a4" +dependencies = [ + "arrow-array", + "rand 0.9.2", +] + +[[package]] +name = "fst" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ab85b9b05e3978cc9a9cf8fea7f01b494e1a09ed3037e16ba39edc7a29eb61a" +dependencies = [ + "utf8-ranges", +] + [[package]] name = "funty" version = "2.0.0" @@ -1873,6 +3070,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "generator" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f04ae4152da20c76fe800fa48659201d5cf627c5149ca0b707b69d7eef6cf9" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -1883,6 +3095,128 @@ dependencies = [ "version_check", ] +[[package]] +name = "geo" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc1a1678e54befc9b4bcab6cd43b8e7f834ae8ea121118b0fd8c42747675b4a" +dependencies = [ + "earcutr", + "float_next_after", + "geo-types", + "geographiclib-rs", + "i_overlay", + "log", + "num-traits", + "robust", + "rstar", + "spade", +] + +[[package]] +name = "geo-traits" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7c353d12a704ccfab1ba8bfb1a7fe6cb18b665bf89d37f4f7890edcd260206" +dependencies = [ + "geo-types", +] + +[[package]] +name = "geo-types" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24f8647af4005fa11da47cd56252c6ef030be8fa97bdbf355e7dfb6348f0a82c" +dependencies = [ + "approx", + "num-traits", + "rayon", + "rstar", + "serde", +] + +[[package]] +name = "geoarrow-array" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1cc4106ac0a0a512c398961ce95d8150475c84a84e17c4511c3643fa120a17" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "geo-traits", + "geoarrow-schema", + "num-traits", + "wkb", + "wkt", +] + +[[package]] +name = "geoarrow-expr-geo" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa84300361ce57fb875bcaa6e32b95b0aff5c6b1af692b936bdd58ff343f4394" +dependencies = [ + "arrow-array", + "arrow-buffer", + "geo", + "geo-traits", + "geoarrow-array", + "geoarrow-schema", +] + +[[package]] +name = "geoarrow-schema" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e97be4e9f523f92bd6a0e0458323f4b783d073d011664decd8dbf05651704f34" +dependencies = [ + "arrow-schema", + "geo-traits", + "serde", + "serde_json", + "thiserror 1.0.69", +] + +[[package]] +name = "geodatafusion" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773cfa1fb0d7f7661b76b3fde00f3ffd8e0ff7b3635096f0ff6294fe5ca62a2b" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-schema", + "datafusion", + "geo", + "geo-traits", + "geoarrow-array", + "geoarrow-expr-geo", + "geoarrow-schema", + "geohash", + "thiserror 1.0.69", + "wkt", +] + +[[package]] +name = "geographiclib-rs" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5a7f08910fd98737a6eda7568e7c5e645093e073328eeef49758cfe8b0489c7" +dependencies = [ + "libm", +] + +[[package]] +name = "geohash" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fb94b1a65401d6cbf22958a9040aa364812c26674f841bee538b12c135db1e6" +dependencies = [ + "geo-types", + "libm", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -1975,6 +3309,27 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy 0.8.37", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -2001,6 +3356,8 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ + "allocator-api2", + "equivalent", "foldhash", "serde", ] @@ -2020,6 +3377,16 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.5.0" @@ -2065,6 +3432,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + [[package]] name = "http" version = "0.2.12" @@ -2138,6 +3511,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + [[package]] name = "hyper" version = "0.14.32" @@ -2285,6 +3664,58 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyperloglogplus" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "621debdf94dcac33e50475fdd76d34d5ea9c0362a834b9db08c3024696c1fbe3" +dependencies = [ + "serde", +] + +[[package]] +name = "i_float" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010025c2c532c8d82e42d0b8bb5184afa449fa6f06c709ea9adcb16c49ae405b" +dependencies = [ + "libm", +] + +[[package]] +name = "i_key_sort" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9190f86706ca38ac8add223b2aed8b1330002b5cdbbce28fb58b10914d38fc27" + +[[package]] +name = "i_overlay" +version = "4.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413183068e6e0289e18d7d0a1f661b81546e6918d5453a44570b9ab30cbed1b3" +dependencies = [ + "i_float", + "i_key_sort", + "i_shape", + "i_tree", + "rayon", +] + +[[package]] +name = "i_shape" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ea154b742f7d43dae2897fcd5ead86bc7b5eefcedd305a7ebf9f69d44d61082" +dependencies = [ + "i_float", +] + +[[package]] +name = "i_tree" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e6d558e6d4c7b82bc51d9c771e7a927862a161a7d87bf2b0541450e0e20915" + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -2508,12 +3939,13 @@ dependencies = [ "deadpool-postgres", "dirs 6.0.0", "dotenvy", - "fs4", + "fs4 0.6.6", "futures", "hkdf", "http-body-util", "hyper 1.8.1", "hyper-util", + "lancedb", "libsql", "mime_guess", "open", @@ -2608,6 +4040,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -2617,6 +4058,24 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -2643,6 +4102,47 @@ dependencies = [ "cc", ] +[[package]] +name = "jiff" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c867c356cc096b33f4981825ab281ecba3db0acefe60329f044c1789d94c6543" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-sys 0.61.2", +] + +[[package]] +name = "jiff-static" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7946b4325269738f270bb55b3c19ab5c5040525f83fd625259422a9d25d9be5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -2663,6 +4163,594 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jsonb" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a901f06163d352fbe41c3c2ff5e08b75330a003cc941e988fb501022f5421e6" +dependencies = [ + "byteorder", + "ethnum", + "fast-float2", + "itoa", + "jiff", + "nom 8.0.0", + "num-traits", + "ordered-float", + "rand 0.9.2", + "ryu", + "serde", + "serde_json", +] + +[[package]] +name = "lance" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b685aca3f97ee02997c83ded16f59c747ccb69e74c8abbbae4aa3d22cf1301" +dependencies = [ + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "async-recursion", + "async-trait", + "async_cell", + "byteorder", + "bytes", + "chrono", + "dashmap", + "datafusion", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-plan", + "deepsize", + "either", + "futures", + "half", + "humantime", + "itertools 0.13.0", + "lance-arrow", + "lance-core", + "lance-datafusion", + "lance-encoding", + "lance-file", + "lance-geo", + "lance-index", + "lance-io", + "lance-linalg", + "lance-namespace", + "lance-table", + "log", + "moka", + "object_store", + "permutation", + "pin-project", + "prost 0.14.3", + "prost-types", + "rand 0.9.2", + "roaring", + "semver", + "serde", + "serde_json", + "snafu", + "tantivy", + "tokio", + "tokio-stream", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "lance-arrow" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf00c7537df524cc518a089f0d156a036d95ca3f5bc2bc1f0a9f9293e9b62ef" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ord", + "arrow-schema", + "arrow-select", + "bytes", + "getrandom 0.2.17", + "half", + "jsonb", + "num-traits", + "rand 0.9.2", +] + +[[package]] +name = "lance-bitpacking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46752e4ac8fc5590a445e780b63a8800adc7a770bd74770a8dc66963778e4e77" +dependencies = [ + "arrayref", + "paste", + "seq-macro", +] + +[[package]] +name = "lance-core" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d13d87d07305c6d4b4dc7780fb1107babf782a0e5b1dc7872e17ae1f8fd11ca" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "async-trait", + "byteorder", + "bytes", + "chrono", + "datafusion-common", + "datafusion-sql", + "deepsize", + "futures", + "itertools 0.13.0", + "lance-arrow", + "libc", + "log", + "mock_instant", + "moka", + "num_cpus", + "object_store", + "pin-project", + "prost 0.14.3", + "rand 0.9.2", + "roaring", + "serde_json", + "snafu", + "tempfile", + "tokio", + "tokio-stream", + "tokio-util", + "tracing", + "url", +] + +[[package]] +name = "lance-datafusion" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6451b5af876eaef8bec4b38a39dadac9d44621e1ecf85d0cdf6097a5d0aa8721" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ord", + "arrow-schema", + "arrow-select", + "async-trait", + "chrono", + "datafusion", + "datafusion-common", + "datafusion-functions", + "datafusion-physical-expr", + "futures", + "jsonb", + "lance-arrow", + "lance-core", + "lance-datagen", + "lance-geo", + "log", + "pin-project", + "prost 0.14.3", + "snafu", + "tokio", + "tracing", +] + +[[package]] +name = "lance-datagen" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1736708dd7867dfbab8fcc930b21c96717c6c00be73b7d9a240336a4ed80375" +dependencies = [ + "arrow", + "arrow-array", + "arrow-cast", + "arrow-schema", + "chrono", + "futures", + "half", + "hex", + "rand 0.9.2", + "rand_distr 0.5.1", + "rand_xoshiro", + "random_word", +] + +[[package]] +name = "lance-encoding" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6b6ca4ff94833240d5ba4a94a742cba786d1949b3c3fa7e11d6f0050443432a" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", + "bytemuck", + "byteorder", + "bytes", + "fsst", + "futures", + "hex", + "hyperloglogplus", + "itertools 0.13.0", + "lance-arrow", + "lance-bitpacking", + "lance-core", + "log", + "lz4", + "num-traits", + "prost 0.14.3", + "prost-build", + "prost-types", + "rand 0.9.2", + "snafu", + "strum", + "tokio", + "tracing", + "xxhash-rust", + "zstd", +] + +[[package]] +name = "lance-file" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55fbe959bffe185543aed3cbeb14484f1aa2e55886034fdb1ea3d8cc9b70aad8" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "async-recursion", + "async-trait", + "byteorder", + "bytes", + "datafusion-common", + "deepsize", + "futures", + "lance-arrow", + "lance-core", + "lance-encoding", + "lance-io", + "log", + "num-traits", + "object_store", + "prost 0.14.3", + "prost-build", + "prost-types", + "snafu", + "tokio", + "tracing", +] + +[[package]] +name = "lance-geo" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a52b0adabc953d457f336a784a3b37353a180e6a79905f544949746e0d4c6483" +dependencies = [ + "datafusion", + "geo-traits", + "geo-types", + "geoarrow-array", + "geoarrow-schema", + "geodatafusion", + "lance-core", + "serde", +] + +[[package]] +name = "lance-index" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b67654bf86fd942dd2cf08294ee7e91053427cd148225f49c9ff398ff9a40fd" +dependencies = [ + "arrow", + "arrow-arith", + "arrow-array", + "arrow-ord", + "arrow-schema", + "arrow-select", + "async-channel", + "async-recursion", + "async-trait", + "bitpacking", + "bitvec", + "bytes", + "crossbeam-queue", + "datafusion", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-sql", + "deepsize", + "dirs 6.0.0", + "fst", + "futures", + "geo-types", + "geoarrow-array", + "geoarrow-schema", + "half", + "itertools 0.13.0", + "jsonb", + "lance-arrow", + "lance-core", + "lance-datafusion", + "lance-datagen", + "lance-encoding", + "lance-file", + "lance-geo", + "lance-io", + "lance-linalg", + "lance-table", + "libm", + "log", + "ndarray", + "num-traits", + "object_store", + "prost 0.14.3", + "prost-build", + "prost-types", + "rand 0.9.2", + "rand_distr 0.5.1", + "rangemap", + "rayon", + "roaring", + "serde", + "serde_json", + "smallvec", + "snafu", + "tantivy", + "tempfile", + "tokio", + "tracing", + "twox-hash", + "uuid", +] + +[[package]] +name = "lance-io" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eb0ccc1c414e31687d83992d546af0a0237c8d2f4bf2ae3d347d539fd0fc141" +dependencies = [ + "arrow", + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", + "async-recursion", + "async-trait", + "byteorder", + "bytes", + "chrono", + "deepsize", + "futures", + "lance-arrow", + "lance-core", + "lance-namespace", + "log", + "object_store", + "path_abs", + "pin-project", + "prost 0.14.3", + "rand 0.9.2", + "serde", + "shellexpand 3.1.1", + "snafu", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "lance-linalg" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "083404cf12dcdb1a7df98fb58f9daf626b6e43a2f794b37b6b89b4012a0e1f78" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "cc", + "deepsize", + "half", + "lance-arrow", + "lance-core", + "num-traits", + "rand 0.9.2", +] + +[[package]] +name = "lance-namespace" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c12778d2aabf9c2bfd16e2509ebe120e562a288d8ae630ec6b6b204868df41b2" +dependencies = [ + "arrow", + "async-trait", + "bytes", + "lance-core", + "lance-namespace-reqwest-client", + "snafu", +] + +[[package]] +name = "lance-namespace-impls" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8863aababdd13a6d2c8d6179dc6981f4f8f49d8b66a00c5dd75115aec4cadc99" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "async-trait", + "bytes", + "chrono", + "futures", + "lance", + "lance-core", + "lance-index", + "lance-io", + "lance-namespace", + "log", + "object_store", + "rand 0.9.2", + "serde_json", + "snafu", + "tokio", + "url", +] + +[[package]] +name = "lance-namespace-reqwest-client" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2acdba67f84190067532fce07b51a435dd390d7cdc1129a05003e5cb3274cf0" +dependencies = [ + "reqwest", + "serde", + "serde_json", + "serde_repr", + "url", +] + +[[package]] +name = "lance-table" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0fcc83f197ce2000c4abe4f5e0873490ab1f41788fa76571c4209b87d4daf50" +dependencies = [ + "arrow", + "arrow-array", + "arrow-buffer", + "arrow-ipc", + "arrow-schema", + "async-trait", + "byteorder", + "bytes", + "chrono", + "deepsize", + "futures", + "lance-arrow", + "lance-core", + "lance-file", + "lance-io", + "log", + "object_store", + "prost 0.14.3", + "prost-build", + "prost-types", + "rand 0.9.2", + "rangemap", + "roaring", + "semver", + "serde", + "serde_json", + "snafu", + "tokio", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "lance-testing" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fb1f7c7e06f91360e141ecee1cf2110f858c231705f69f2cd2fda9e30c1e9f4" +dependencies = [ + "arrow-array", + "arrow-schema", + "lance-arrow", + "num-traits", + "rand 0.9.2", +] + +[[package]] +name = "lancedb" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23f89be66655e426c6afac36032ede6d7213248462329dd73f9ac6bd1cc370b8" +dependencies = [ + "ahash 0.8.12", + "arrow", + "arrow-array", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-schema", + "arrow-select", + "async-trait", + "bytes", + "chrono", + "datafusion", + "datafusion-catalog", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", + "datafusion-physical-plan", + "futures", + "half", + "lance", + "lance-arrow", + "lance-core", + "lance-datafusion", + "lance-datagen", + "lance-encoding", + "lance-file", + "lance-index", + "lance-io", + "lance-linalg", + "lance-namespace", + "lance-namespace-impls", + "lance-table", + "lance-testing", + "lazy_static", + "log", + "moka", + "num-traits", + "object_store", + "pin-project", + "rand 0.9.2", + "regex", + "semver", + "serde", + "serde_json", + "serde_with", + "snafu", + "tempfile", + "tokio", + "url", + "uuid", +] + [[package]] name = "lazy-regex" version = "3.5.1" @@ -2710,6 +4798,69 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + [[package]] name = "libc" version = "0.2.180" @@ -2846,7 +4997,7 @@ dependencies = [ "libsql-rusqlite", "libsql-sys", "parking_lot", - "prost", + "prost 0.12.6", "serde", "thiserror 1.0.69", "tokio", @@ -2907,12 +5058,68 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "lru-slab" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lz4" +version = "1.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4" +dependencies = [ + "lz4-sys", +] + +[[package]] +name = "lz4-sys" +version = "1.11.1+lz4-1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "lz4_flex" +version = "0.11.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" + +[[package]] +name = "lz4_flex" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" +dependencies = [ + "twox-hash", +] + [[package]] name = "mach2" version = "0.4.3" @@ -2943,6 +5150,19 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "num_cpus", + "once_cell", + "rawpointer", + "thread-tree", +] + [[package]] name = "maybe-owned" version = "0.3.4" @@ -2959,6 +5179,15 @@ dependencies = [ "digest", ] +[[package]] +name = "measure_time" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51c55d61e72fc3ab704396c5fa16f4c184db37978ae4e94ca8959693a235fc0e" +dependencies = [ + "log", +] + [[package]] name = "memchr" version = "2.7.6" @@ -2974,6 +5203,15 @@ dependencies = [ "rustix 1.1.3", ] +[[package]] +name = "memmap2" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714098028fe011992e1c3962653c96b2d578c4b4bce9036e15ff220319b1e0e3" +dependencies = [ + "libc", +] + [[package]] name = "memoffset" version = "0.9.1" @@ -3014,6 +5252,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.1.1" @@ -3026,6 +5274,44 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "mock_instant" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce6dd36094cac388f119d2e9dc82dc730ef91c32a6222170d630e5414b956e6" + +[[package]] +name = "moka" +version = "0.12.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac832c50ced444ef6be0767a008b02c106a909ba79d1d830501e94b96f6b7e" +dependencies = [ + "async-lock", + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "event-listener", + "futures-util", + "parking_lot", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + [[package]] name = "nanoid" version = "0.4.0" @@ -3052,6 +5338,21 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "nibble_vec" version = "0.1.0" @@ -3096,6 +5397,15 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -3182,6 +5492,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -3194,6 +5505,28 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "object" version = "0.36.7" @@ -3215,6 +5548,30 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbfbfff40aeccab00ec8a910b57ca8ecf4319b335c542f2edcd19dd25a1e2a00" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "futures", + "http 1.4.0", + "humantime", + "itertools 0.14.0", + "parking_lot", + "percent-encoding", + "thiserror 2.0.18", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -3227,6 +5584,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oneshot" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -3319,6 +5682,15 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "ownedbytes" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fbd56f7631767e61784dc43f8580f403f4475bd4aaa4da003e6295e1bab4a7e" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "parking" version = "2.2.1" @@ -3379,6 +5751,18 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +[[package]] +name = "path_abs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05ef02f6342ac01d8a93b65f96db53fe68a92a15f41144f97fb00a9e669633c3" +dependencies = [ + "serde", + "serde_derive", + "std_prelude", + "stfu8", +] + [[package]] name = "pathdiff" version = "0.2.3" @@ -3397,6 +5781,24 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "permutation" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df202b0b0f5b8e389955afd5f27b007b00fb948162953f1db9c70d2c7e3157d7" + +[[package]] +name = "petgraph" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" +dependencies = [ + "fixedbitset", + "hashbrown 0.15.5", + "indexmap 2.13.0", + "serde", +] + [[package]] name = "pgvector" version = "0.4.1" @@ -3416,6 +5818,15 @@ dependencies = [ "phf_shared 0.11.3", ] +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", +] + [[package]] name = "phf" version = "0.13.1" @@ -3456,6 +5867,15 @@ dependencies = [ "uncased", ] +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + [[package]] name = "phf_shared" version = "0.13.1" @@ -3540,6 +5960,21 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a9db96d7fa8782dd8c15ce32ffe8680bbd1e978a43bf51a34d39483540495f5" +dependencies = [ + "portable-atomic", +] + [[package]] name = "postcard" version = "1.1.3" @@ -3654,7 +6089,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.12.6", +] + +[[package]] +name = "prost" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2ea70524a2f82d518bce41317d0fae74151505651af45faf1ffbd6fd33f0568" +dependencies = [ + "bytes", + "prost-derive 0.14.3", +] + +[[package]] +name = "prost-build" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" +dependencies = [ + "heck", + "itertools 0.12.1", + "log", + "multimap", + "petgraph", + "prettyplease", + "prost 0.14.3", + "prost-types", + "regex", + "syn 2.0.114", + "tempfile", ] [[package]] @@ -3664,12 +6128,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.114", ] +[[package]] +name = "prost-derive" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" +dependencies = [ + "anyhow", + "itertools 0.12.1", + "proc-macro2", + "quote", + "syn 2.0.114", +] + +[[package]] +name = "prost-types" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8991c4cbdb8bc5b11f0b074ffe286c30e523de90fee5ba8132f1399f23cb3dd7" +dependencies = [ + "prost 0.14.3", +] + [[package]] name = "psm" version = "0.1.29" @@ -3856,6 +6342,60 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand 0.8.5", +] + +[[package]] +name = "rand_distr" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463" +dependencies = [ + "num-traits", + "rand 0.9.2", +] + +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "random_word" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e47a395bdb55442b883c89062d6bcff25dc90fa5f8369af81e0ac6d49d78cf81" +dependencies = [ + "ahash 0.8.12", + "brotli", + "paste", + "rand 0.9.2", + "unicase", +] + +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "rayon" version = "1.11.0" @@ -4033,6 +6573,12 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "rend" version = "0.4.2" @@ -4165,6 +6711,72 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "roaring" +version = "0.10.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19e8d2cfa184d94d0726d650a9f4a1be7f9b76ac9fdb954219878dc00c1c1e7b" +dependencies = [ + "bytemuck", + "byteorder", +] + +[[package]] +name = "robust" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e27ee8bb91ca0adcf0ecb116293afa12d393f9c2b9b9cd54d33e8078fe19839" + +[[package]] +name = "rstar" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "421400d13ccfd26dfa5858199c30a5d76f9c54e0dba7575273025b43c5175dbb" +dependencies = [ + "heapless", + "num-traits", + "smallvec", +] + +[[package]] +name = "rstest" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", +] + +[[package]] +name = "rstest_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn 2.0.114", + "unicode-ident", +] + +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + [[package]] name = "rust_decimal" version = "1.40.0" @@ -4412,6 +7024,12 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -4499,6 +7117,12 @@ dependencies = [ "serde_core", ] +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + [[package]] name = "serde" version = "1.0.228" @@ -4682,6 +7306,15 @@ dependencies = [ "dirs 4.0.0", ] +[[package]] +name = "shellexpand" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1fdf65dd6331831494dd616b30351c38e96e45921a27745cf98490458b90bb" +dependencies = [ + "dirs 6.0.0", +] + [[package]] name = "shlex" version = "1.3.0" @@ -4719,6 +7352,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + [[package]] name = "simdutf8" version = "0.1.5" @@ -4731,6 +7370,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2aa850e253778c88a04c3d7323b043aeda9d3e30d5971937c1855769763678e" +[[package]] +name = "sketches-ddsketch" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a" +dependencies = [ + "serde", +] + [[package]] name = "slab" version = "0.4.12" @@ -4746,6 +7394,27 @@ dependencies = [ "serde", ] +[[package]] +name = "snafu" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e84b3f4eacbf3a1ce05eac6763b4d629d60cbc94d632e4092c54ade71f1e1a2" +dependencies = [ + "snafu-derive", +] + +[[package]] +name = "snafu-derive" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c97747dbf44bb1ca44a561ece23508e99cb592e862f22222dcf42f51d1e451" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "socket2" version = "0.5.10" @@ -4766,12 +7435,45 @@ dependencies = [ "windows-sys 0.60.2", ] +[[package]] +name = "spade" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb313e1c8afee5b5647e00ee0fe6855e3d529eb863a0fdae1d60006c4d1e9990" +dependencies = [ + "hashbrown 0.15.5", + "num-traits", + "robust", + "smallvec", +] + [[package]] name = "sptr" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b9b39299b249ad65f3b7e96443bad61c02ca5cd3589f46cb6d610a0fd6c0d6a" +[[package]] +name = "sqlparser" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4591acadbcf52f0af60eafbb2c003232b2b4cd8de5f0e9437cb8b1b59046cc0f" +dependencies = [ + "log", + "sqlparser_derive", +] + +[[package]] +name = "sqlparser_derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5fc6819faabb412da764b99d3b713bb55083c11e7e0c00144d386cd6a1939c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.114", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -4784,6 +7486,18 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "std_prelude" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8207e78455ffdf55661170876f88daf85356e4edd54e0a3dbc79586ca1e50cbe" + +[[package]] +name = "stfu8" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e51f1e89f093f99e7432c491c382b88a6860a5adbe6bf02574bf0a08efff1978" + [[package]] name = "strict" version = "0.2.0" @@ -4830,6 +7544,28 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.114", +] + [[package]] name = "subtle" version = "2.6.1" @@ -4921,6 +7657,158 @@ dependencies = [ "winx", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + +[[package]] +name = "tantivy" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a966cb0e76e311f09cf18507c9af192f15d34886ee43d7ba7c7e3803660c43" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64 0.22.1", + "bitpacking", + "bon", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "downcast-rs", + "fastdivide", + "fnv", + "fs4 0.8.4", + "htmlescape", + "hyperloglogplus", + "itertools 0.14.0", + "levenshtein_automata", + "log", + "lru", + "lz4_flex 0.11.5", + "measure_time", + "memmap2", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash 2.1.1", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror 2.0.18", + "time", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1adc286a39e089ae9938935cd488d7d34f14502544a36607effd2239ff0e2494" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6300428e0c104c4f7db6f95b466a6f5c1b9aece094ec57cdd365337908dc7344" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools 0.14.0", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b6ea6090ce03dc72c27d0619e77185d26cc3b20775966c346c6d4f7e99d7f" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e810cdeeebca57fc3f7bfec5f85fdbea9031b2ac9b990eb5ff49b371d52bbe6a" +dependencies = [ + "nom 7.1.3", + "serde", + "serde_json", +] + +[[package]] +name = "tantivy-sstable" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709f22c08a4c90e1b36711c1c6cad5ae21b20b093e535b69b18783dd2cb99416" +dependencies = [ + "futures-util", + "itertools 0.14.0", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bcdebb267671311d1e8891fd9d1301803fdb8ad21ba22e0a30d0cab49ba59c1" +dependencies = [ + "murmurhash32", + "rand_distr 0.4.3", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfa942fcee81e213e09715bbce8734ae2180070b97b33839a795ba1de201547d" +dependencies = [ + "serde", +] + [[package]] name = "tap" version = "1.0.1" @@ -5049,6 +7937,15 @@ dependencies = [ "syn 2.0.114", ] +[[package]] +name = "thread-tree" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffbd370cb847953a25954d9f63e14824a36113f8c72eecf6eccef5dc4b45d630" +dependencies = [ + "crossbeam-channel", +] + [[package]] name = "thread_local" version = "1.1.9" @@ -5089,6 +7986,15 @@ dependencies = [ "time-core", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tinystr" version = "0.8.2" @@ -5363,7 +8269,7 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost", + "prost 0.12.6", "tokio", "tokio-stream", "tower 0.4.13", @@ -5454,13 +8360,18 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ + "async-compression", "bitflags 2.10.0", "bytes", + "futures-core", "futures-util", "http 1.4.0", "http-body 1.0.1", + "http-body-util", "iri-string", "pin-project-lite", + "tokio", + "tokio-util", "tower 0.5.3", "tower-layer", "tower-service", @@ -5606,6 +8517,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +dependencies = [ + "rand 0.9.2", +] + [[package]] name = "typenum" version = "1.19.0" @@ -5730,6 +8650,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -6086,7 +9012,7 @@ dependencies = [ "cranelift-frontend", "cranelift-native", "gimli", - "itertools", + "itertools 0.12.1", "log", "object 0.36.7", "smallvec", @@ -6337,7 +9263,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "shellexpand", + "shellexpand 2.1.2", "syn 2.0.114", "witx", ] @@ -6758,6 +9684,31 @@ dependencies = [ "wast 35.0.2", ] +[[package]] +name = "wkb" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a120b336c7ad17749026d50427c23d838ecb50cd64aaea6254b5030152f890a9" +dependencies = [ + "byteorder", + "geo-traits", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "wkt" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efb2b923ccc882312e559ffaa832a055ba9d1ac0cc8e86b3e25453247e4b81d7" +dependencies = [ + "geo-traits", + "geo-types", + "log", + "num-traits", + "thiserror 1.0.69", +] + [[package]] name = "writeable" version = "0.6.2" @@ -6793,6 +9744,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index e25b0e5e..cb92053f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,6 +108,9 @@ open = "5" # The postgres feature provides ToSql/FromSql for postgres-types (shared by tokio-postgres) pgvector = { version = "0.4", features = ["postgres"], optional = true } +# LanceDB vector store (optional alternative to pgvector/libsql for workspace search) +lancedb = { version = "0.26", optional = true } + # WASM sandbox for untrusted tool execution wasmtime = { version = "28", features = ["component-model"] } wasmtime-wasi = "28" # WASI support for component model @@ -153,6 +156,7 @@ tempfile = "3" [features] default = ["postgres", "libsql"] +lancedb = ["dep:lancedb"] postgres = [ "dep:deadpool-postgres", "dep:tokio-postgres", diff --git a/FEATURE_PARITY.md b/FEATURE_PARITY.md index 9ce349f0..15a7b955 100644 --- a/FEATURE_PARITY.md +++ b/FEATURE_PARITY.md @@ -251,7 +251,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Gemini embeddings | ✅ | ❌ | | | Local embeddings | ✅ | ❌ | | | SQLite-vec backend | ✅ | ❌ | IronClaw uses PostgreSQL | -| LanceDB backend | ✅ | ❌ | | +| LanceDB backend | ✅ | ✅ | VectorStore + LanceDbVectorStore, DbWithLanceVectorStore wrapper, VECTOR_BACKEND=lancedb | | QMD backend | ✅ | ❌ | | | Atomic reindexing | ✅ | ✅ | | | Embeddings batching | ✅ | ❌ | | diff --git a/src/config.rs b/src/config.rs index b273c413..6f5d5dc6 100644 --- a/src/config.rs +++ b/src/config.rs @@ -310,6 +310,33 @@ impl std::str::FromStr for DatabaseBackend { } } +/// Which vector store to use for workspace semantic search. +/// +/// When None, uses the database backend's built-in (pgvector or libsql_vector_idx). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum VectorBackend { + /// Use database built-in (pgvector or libsql_vector_idx). + #[default] + Builtin, + /// Use LanceDB for vector search (requires feature "lancedb"). + LanceDb, +} + +impl std::str::FromStr for VectorBackend { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "builtin" | "pgvector" | "libsql" | "" => Ok(Self::Builtin), + "lancedb" | "lance" => Ok(Self::LanceDb), + _ => Err(format!( + "invalid vector backend '{}', expected 'builtin' or 'lancedb'", + s + )), + } + } +} + /// Database configuration. #[derive(Debug, Clone)] pub struct DatabaseConfig { @@ -327,6 +354,12 @@ pub struct DatabaseConfig { pub libsql_url: Option, /// Turso auth token (required when libsql_url is set). pub libsql_auth_token: Option, + + // -- Vector store (workspace semantic search) -- + /// Override vector backend (default: use database built-in). + pub vector_backend: VectorBackend, + /// Path for LanceDB when vector_backend=LanceDb (default: ~/.ironclaw/lancedb). + pub lancedb_path: Option, } impl DatabaseConfig { @@ -376,6 +409,19 @@ impl DatabaseConfig { }); } + let vector_backend: VectorBackend = + optional_env("VECTOR_BACKEND")? + .and_then(|s| s.parse().ok()) + .unwrap_or_default(); + + let lancedb_path = optional_env("LANCEDB_PATH")?.map(PathBuf::from).or_else(|| { + if vector_backend == VectorBackend::LanceDb { + Some(default_lancedb_path()) + } else { + None + } + }); + Ok(Self { backend, url: SecretString::from(url), @@ -383,6 +429,8 @@ impl DatabaseConfig { libsql_path, libsql_url, libsql_auth_token, + vector_backend, + lancedb_path, }) } @@ -400,6 +448,39 @@ pub fn default_libsql_path() -> PathBuf { .join("ironclaw.db") } +/// Default LanceDB path (~/.ironclaw/lancedb). +pub fn default_lancedb_path() -> PathBuf { + dirs::home_dir() + .unwrap_or_else(|| PathBuf::from(".")) + .join(".ironclaw") + .join("lancedb") +} + +#[cfg(test)] +mod tests { + use super::VectorBackend; + + #[test] + fn test_vector_backend_parse() { + assert_eq!("builtin".parse::().unwrap(), VectorBackend::Builtin); + assert_eq!("pgvector".parse::().unwrap(), VectorBackend::Builtin); + assert_eq!("libsql".parse::().unwrap(), VectorBackend::Builtin); + assert_eq!("".parse::().unwrap(), VectorBackend::Builtin); + + assert_eq!("lancedb".parse::().unwrap(), VectorBackend::LanceDb); + assert_eq!("lance".parse::().unwrap(), VectorBackend::LanceDb); + assert_eq!("Lancedb".parse::().unwrap(), VectorBackend::LanceDb); + + assert!("invalid".parse::().is_err()); + } + + #[test] + fn test_default_lancedb_path() { + let path = super::default_lancedb_path(); + assert!(path.to_string_lossy().ends_with(".ironclaw/lancedb")); + } +} + /// Which LLM backend to use. /// /// Defaults to `NearAi` to keep IronClaw close to the NEAR ecosystem. diff --git a/src/db/lancedb_wrapper.rs b/src/db/lancedb_wrapper.rs new file mode 100644 index 00000000..d7b089e3 --- /dev/null +++ b/src/db/lancedb_wrapper.rs @@ -0,0 +1,664 @@ +//! Database wrapper that uses LanceDB for vector search. +//! +//! When `VECTOR_BACKEND=lancedb`, this wraps the main database (Postgres or libSQL) +//! and delegates vector search to LanceDB while using the inner DB for FTS and all +//! other operations. + +use std::sync::Arc; + +use async_trait::async_trait; +use uuid::Uuid; + +use crate::agent::BrokenTool; +use crate::agent::routine::{Routine, RoutineRun, RunStatus}; +use crate::context::{ActionRecord, JobContext, JobState}; +use crate::db::Database; +use crate::error::{DatabaseError, WorkspaceError}; +use crate::history::{ + ConversationMessage, ConversationSummary, JobEventRecord, LlmCallRecord, SandboxJobRecord, + SandboxJobSummary, SettingRow, +}; +use crate::workspace::{ + MemoryChunk, MemoryDocument, SearchConfig, SearchResult, WorkspaceEntry, +}; +use crate::workspace::search::{RankedResult, reciprocal_rank_fusion}; +use crate::workspace::lancedb_store::VectorStore; + +/// Wraps a Database with a LanceDB VectorStore for hybrid search. +/// +/// Documents and chunks stay in the inner DB. Vector search uses LanceDB. +pub struct DbWithLanceVectorStore { + inner: Arc, + vector_store: Arc, +} + +impl DbWithLanceVectorStore { + pub fn new(inner: Arc, vector_store: Arc) -> Self { + Self { inner, vector_store } + } +} + +#[async_trait] +impl Database for DbWithLanceVectorStore { + async fn run_migrations(&self) -> Result<(), DatabaseError> { + self.inner.run_migrations().await + } + + async fn create_conversation( + &self, + channel: &str, + user_id: &str, + thread_id: Option<&str>, + ) -> Result { + self.inner + .create_conversation(channel, user_id, thread_id) + .await + } + + async fn touch_conversation(&self, id: Uuid) -> Result<(), DatabaseError> { + self.inner.touch_conversation(id).await + } + + async fn add_conversation_message( + &self, + conversation_id: Uuid, + role: &str, + content: &str, + ) -> Result { + self.inner + .add_conversation_message(conversation_id, role, content) + .await + } + + async fn ensure_conversation( + &self, + id: Uuid, + channel: &str, + user_id: &str, + thread_id: Option<&str>, + ) -> Result<(), DatabaseError> { + self.inner + .ensure_conversation(id, channel, user_id, thread_id) + .await + } + + async fn list_conversations_with_preview( + &self, + user_id: &str, + channel: &str, + limit: i64, + ) -> Result, DatabaseError> { + self.inner + .list_conversations_with_preview(user_id, channel, limit) + .await + } + + async fn get_or_create_assistant_conversation( + &self, + user_id: &str, + channel: &str, + ) -> Result { + self.inner + .get_or_create_assistant_conversation(user_id, channel) + .await + } + + async fn create_conversation_with_metadata( + &self, + user_id: &str, + channel: &str, + metadata: &serde_json::Value, + ) -> Result { + self.inner + .create_conversation_with_metadata(user_id, channel, metadata) + .await + } + + async fn list_conversation_messages_paginated( + &self, + conversation_id: Uuid, + before: Option>, + limit: i64, + ) -> Result<(Vec, bool), DatabaseError> { + self.inner + .list_conversation_messages_paginated(conversation_id, before, limit) + .await + } + + async fn update_conversation_metadata_field( + &self, + id: Uuid, + key: &str, + value: &serde_json::Value, + ) -> Result<(), DatabaseError> { + self.inner + .update_conversation_metadata_field(id, key, value) + .await + } + + async fn get_conversation_metadata( + &self, + id: Uuid, + ) -> Result, DatabaseError> { + self.inner.get_conversation_metadata(id).await + } + + async fn list_conversation_messages( + &self, + conversation_id: Uuid, + ) -> Result, DatabaseError> { + self.inner.list_conversation_messages(conversation_id).await + } + + async fn conversation_belongs_to_user( + &self, + conversation_id: Uuid, + user_id: &str, + ) -> Result { + self.inner + .conversation_belongs_to_user(conversation_id, user_id) + .await + } + + async fn save_job(&self, ctx: &JobContext) -> Result<(), DatabaseError> { + self.inner.save_job(ctx).await + } + + async fn get_job(&self, id: Uuid) -> Result, DatabaseError> { + self.inner.get_job(id).await + } + + async fn update_job_status( + &self, + id: Uuid, + status: crate::context::JobState, + failure_reason: Option<&str>, + ) -> Result<(), DatabaseError> { + self.inner.update_job_status(id, status, failure_reason).await + } + + async fn mark_job_stuck(&self, id: Uuid) -> Result<(), DatabaseError> { + self.inner.mark_job_stuck(id).await + } + + async fn get_stuck_jobs(&self) -> Result, DatabaseError> { + self.inner.get_stuck_jobs().await + } + + async fn save_action(&self, job_id: Uuid, action: &ActionRecord) -> Result<(), DatabaseError> { + self.inner.save_action(job_id, action).await + } + + async fn get_job_actions(&self, job_id: Uuid) -> Result, DatabaseError> { + self.inner.get_job_actions(job_id).await + } + + async fn record_llm_call(&self, record: &LlmCallRecord<'_>) -> Result { + self.inner.record_llm_call(record).await + } + + async fn save_estimation_snapshot( + &self, + job_id: Uuid, + category: &str, + tool_names: &[String], + estimated_cost: rust_decimal::Decimal, + estimated_time_secs: i32, + estimated_value: rust_decimal::Decimal, + ) -> Result { + self.inner + .save_estimation_snapshot( + job_id, + category, + tool_names, + estimated_cost, + estimated_time_secs, + estimated_value, + ) + .await + } + + async fn update_estimation_actuals( + &self, + id: Uuid, + actual_cost: rust_decimal::Decimal, + actual_time_secs: i32, + actual_value: Option, + ) -> Result<(), DatabaseError> { + self.inner + .update_estimation_actuals(id, actual_cost, actual_time_secs, actual_value) + .await + } + + async fn save_sandbox_job(&self, job: &SandboxJobRecord) -> Result<(), DatabaseError> { + self.inner.save_sandbox_job(job).await + } + + async fn get_sandbox_job(&self, id: Uuid) -> Result, DatabaseError> { + self.inner.get_sandbox_job(id).await + } + + async fn list_sandbox_jobs(&self) -> Result, DatabaseError> { + self.inner.list_sandbox_jobs().await + } + + async fn update_sandbox_job_status( + &self, + id: Uuid, + status: &str, + success: Option, + message: Option<&str>, + started_at: Option>, + completed_at: Option>, + ) -> Result<(), DatabaseError> { + self.inner + .update_sandbox_job_status(id, status, success, message, started_at, completed_at) + .await + } + + async fn cleanup_stale_sandbox_jobs(&self) -> Result { + self.inner.cleanup_stale_sandbox_jobs().await + } + + async fn sandbox_job_summary(&self) -> Result { + self.inner.sandbox_job_summary().await + } + + async fn list_sandbox_jobs_for_user( + &self, + user_id: &str, + ) -> Result, DatabaseError> { + self.inner.list_sandbox_jobs_for_user(user_id).await + } + + async fn sandbox_job_summary_for_user( + &self, + user_id: &str, + ) -> Result { + self.inner.sandbox_job_summary_for_user(user_id).await + } + + async fn sandbox_job_belongs_to_user( + &self, + job_id: Uuid, + user_id: &str, + ) -> Result { + self.inner + .sandbox_job_belongs_to_user(job_id, user_id) + .await + } + + async fn update_sandbox_job_mode(&self, id: Uuid, mode: &str) -> Result<(), DatabaseError> { + self.inner.update_sandbox_job_mode(id, mode).await + } + + async fn get_sandbox_job_mode(&self, id: Uuid) -> Result, DatabaseError> { + self.inner.get_sandbox_job_mode(id).await + } + + async fn save_job_event( + &self, + job_id: Uuid, + event_type: &str, + data: &serde_json::Value, + ) -> Result<(), DatabaseError> { + self.inner.save_job_event(job_id, event_type, data).await + } + + async fn list_job_events( + &self, + job_id: Uuid, + limit: Option, + ) -> Result, DatabaseError> { + self.inner.list_job_events(job_id, limit).await + } + + async fn create_routine(&self, routine: &crate::agent::routine::Routine) -> Result<(), DatabaseError> { + self.inner.create_routine(routine).await + } + + async fn get_routine(&self, id: Uuid) -> Result, DatabaseError> { + self.inner.get_routine(id).await + } + + async fn get_routine_by_name( + &self, + user_id: &str, + name: &str, + ) -> Result, DatabaseError> { + self.inner.get_routine_by_name(user_id, name).await + } + + async fn list_routines( + &self, + user_id: &str, + ) -> Result, DatabaseError> { + self.inner.list_routines(user_id).await + } + + async fn list_event_routines(&self) -> Result, DatabaseError> { + self.inner.list_event_routines().await + } + + async fn list_due_cron_routines(&self) -> Result, DatabaseError> { + self.inner.list_due_cron_routines().await + } + + async fn update_routine(&self, routine: &crate::agent::routine::Routine) -> Result<(), DatabaseError> { + self.inner.update_routine(routine).await + } + + async fn update_routine_runtime( + &self, + id: Uuid, + last_run_at: chrono::DateTime, + next_fire_at: Option>, + run_count: u64, + consecutive_failures: u32, + state: &serde_json::Value, + ) -> Result<(), DatabaseError> { + self.inner + .update_routine_runtime( + id, + last_run_at, + next_fire_at, + run_count, + consecutive_failures, + state, + ) + .await + } + + async fn delete_routine(&self, id: Uuid) -> Result { + self.inner.delete_routine(id).await + } + + async fn create_routine_run(&self, run: &RoutineRun) -> Result { + self.inner.create_routine_run(run).await + } + + async fn complete_routine_run( + &self, + id: Uuid, + status: RunStatus, + result_summary: Option<&str>, + tokens_used: Option, + ) -> Result<(), DatabaseError> { + self.inner + .complete_routine_run(id, status, result_summary, tokens_used) + .await + } + + async fn list_routine_runs( + &self, + routine_id: Uuid, + limit: i64, + ) -> Result, DatabaseError> { + self.inner.list_routine_runs(routine_id, limit).await + } + + async fn count_running_routine_runs(&self, routine_id: Uuid) -> Result { + self.inner.count_running_routine_runs(routine_id).await + } + + async fn record_tool_failure( + &self, + tool_name: &str, + error_message: &str, + ) -> Result<(), DatabaseError> { + self.inner.record_tool_failure(tool_name, error_message).await + } + + async fn get_broken_tools(&self, threshold: i32) -> Result, DatabaseError> { + self.inner.get_broken_tools(threshold).await + } + + async fn mark_tool_repaired(&self, tool_name: &str) -> Result<(), DatabaseError> { + self.inner.mark_tool_repaired(tool_name).await + } + + async fn increment_repair_attempts(&self, tool_name: &str) -> Result<(), DatabaseError> { + self.inner.increment_repair_attempts(tool_name).await + } + + async fn get_setting( + &self, + user_id: &str, + key: &str, + ) -> Result, DatabaseError> { + self.inner.get_setting(user_id, key).await + } + + async fn get_setting_full( + &self, + user_id: &str, + key: &str, + ) -> Result, DatabaseError> { + self.inner.get_setting_full(user_id, key).await + } + + async fn set_setting( + &self, + user_id: &str, + key: &str, + value: &serde_json::Value, + ) -> Result<(), DatabaseError> { + self.inner.set_setting(user_id, key, value).await + } + + async fn delete_setting(&self, user_id: &str, key: &str) -> Result { + self.inner.delete_setting(user_id, key).await + } + + async fn list_settings(&self, user_id: &str) -> Result, DatabaseError> { + self.inner.list_settings(user_id).await + } + + async fn get_all_settings( + &self, + user_id: &str, + ) -> Result, DatabaseError> { + self.inner.get_all_settings(user_id).await + } + + async fn set_all_settings( + &self, + user_id: &str, + settings: &std::collections::HashMap, + ) -> Result<(), DatabaseError> { + self.inner.set_all_settings(user_id, settings).await + } + + async fn has_settings(&self, user_id: &str) -> Result { + self.inner.has_settings(user_id).await + } + + async fn get_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result { + self.inner + .get_document_by_path(user_id, agent_id, path) + .await + } + + async fn get_document_by_id(&self, id: Uuid) -> Result { + self.inner.get_document_by_id(id).await + } + + async fn get_or_create_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result { + self.inner + .get_or_create_document_by_path(user_id, agent_id, path) + .await + } + + async fn update_document(&self, id: Uuid, content: &str) -> Result<(), WorkspaceError> { + self.inner.update_document(id, content).await + } + + async fn delete_document_by_path( + &self, + user_id: &str, + agent_id: Option, + path: &str, + ) -> Result<(), WorkspaceError> { + self.inner + .delete_document_by_path(user_id, agent_id, path) + .await + } + + async fn list_directory( + &self, + user_id: &str, + agent_id: Option, + directory: &str, + ) -> Result, WorkspaceError> { + self.inner + .list_directory(user_id, agent_id, directory) + .await + } + + async fn list_all_paths( + &self, + user_id: &str, + agent_id: Option, + ) -> Result, WorkspaceError> { + self.inner.list_all_paths(user_id, agent_id).await + } + + async fn list_documents( + &self, + user_id: &str, + agent_id: Option, + ) -> Result, WorkspaceError> { + self.inner.list_documents(user_id, agent_id).await + } + + async fn delete_chunks(&self, document_id: Uuid) -> Result<(), WorkspaceError> { + self.inner.delete_chunks(document_id).await?; + self.vector_store.delete_chunks(document_id).await?; + Ok(()) + } + + async fn insert_chunk( + &self, + document_id: Uuid, + chunk_index: i32, + content: &str, + embedding: Option<&[f32]>, + ) -> Result { + let chunk_id = self + .inner + .insert_chunk(document_id, chunk_index, content, embedding) + .await?; + + if let Some(emb) = embedding { + let doc = self.inner.get_document_by_id(document_id).await?; + self.vector_store + .insert_chunk( + chunk_id, + document_id, + &doc.user_id, + doc.agent_id, + content, + emb, + ) + .await?; + } + + Ok(chunk_id) + } + + async fn update_chunk_embedding( + &self, + chunk_id: Uuid, + embedding: &[f32], + ) -> Result<(), WorkspaceError> { + self.inner.update_chunk_embedding(chunk_id, embedding).await?; + + if let Some(chunk) = self.inner.get_chunk_by_id(chunk_id).await? { + let doc = self.inner.get_document_by_id(chunk.document_id).await?; + self.vector_store + .update_chunk_embedding( + chunk_id, + chunk.document_id, + &doc.user_id, + doc.agent_id, + &chunk.content, + embedding, + ) + .await?; + } + + Ok(()) + } + + async fn get_chunks_without_embeddings( + &self, + user_id: &str, + agent_id: Option, + limit: usize, + ) -> Result, WorkspaceError> { + self.inner + .get_chunks_without_embeddings(user_id, agent_id, limit) + .await + } + + async fn get_chunk_by_id( + &self, + chunk_id: Uuid, + ) -> Result, WorkspaceError> { + self.inner.get_chunk_by_id(chunk_id).await + } + + async fn hybrid_search( + &self, + user_id: &str, + agent_id: Option, + query: &str, + embedding: Option<&[f32]>, + config: &SearchConfig, + ) -> Result, WorkspaceError> { + let fts_config = SearchConfig { + use_fts: true, + use_vector: false, + ..*config + }; + let fts_search_results = self + .inner + .hybrid_search(user_id, agent_id, query, None, &fts_config) + .await?; + + let fts_results: Vec = fts_search_results + .iter() + .enumerate() + .map(|(i, r)| RankedResult { + chunk_id: r.chunk_id, + document_id: r.document_id, + content: r.content.clone(), + rank: (i + 1) as u32, + }) + .collect(); + + let vector_results = if config.use_vector { + if let Some(emb) = embedding { + self.vector_store + .vector_search(user_id, agent_id, emb, config.pre_fusion_limit) + .await? + } else { + Vec::new() + } + } else { + Vec::new() + }; + + Ok(reciprocal_rank_fusion(fts_results, vector_results, config)) + } +} diff --git a/src/db/libsql_backend.rs b/src/db/libsql_backend.rs index 31f79e77..827fcc43 100644 --- a/src/db/libsql_backend.rs +++ b/src/db/libsql_backend.rs @@ -2486,6 +2486,46 @@ impl Database for LibSqlBackend { Ok(chunks) } + async fn get_chunk_by_id( + &self, + chunk_id: Uuid, + ) -> Result, WorkspaceError> { + let conn = self + .connect() + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: e.to_string(), + })?; + let mut rows = conn + .query( + r#" + SELECT id, document_id, chunk_index, content, created_at + FROM memory_chunks WHERE id = ?1 + "#, + params![chunk_id.to_string()], + ) + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Query failed: {}", e), + })?; + + let row = rows + .next() + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Row fetch failed: {}", e), + })?; + + Ok(row.map(|r| MemoryChunk { + id: get_text(&r, 0).parse().unwrap_or_default(), + document_id: get_text(&r, 1).parse().unwrap_or_default(), + chunk_index: get_i64(&r, 2) as i32, + content: get_text(&r, 3), + embedding: None, + created_at: get_ts(&r, 4), + })) + } + // ==================== Workspace: Search ==================== async fn hybrid_search( diff --git a/src/db/mod.rs b/src/db/mod.rs index 40e85cae..6b9136e4 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -18,6 +18,9 @@ pub mod libsql_backend; #[cfg(feature = "libsql")] pub mod libsql_migrations; +#[cfg(all(feature = "lancedb", any(feature = "postgres", feature = "libsql")))] +pub mod lancedb_wrapper; + use std::collections::HashMap; use std::sync::Arc; @@ -48,7 +51,7 @@ use crate::workspace::{SearchConfig, SearchResult}; pub async fn connect_from_config( config: &crate::config::DatabaseConfig, ) -> Result, DatabaseError> { - match config.backend { + let inner: Arc = match config.backend { #[cfg(feature = "libsql")] crate::config::DatabaseBackend::LibSql => { use secrecy::ExposeSecret as _; @@ -75,7 +78,7 @@ pub async fn connect_from_config( .map_err(|e| DatabaseError::Pool(e.to_string()))? }; backend.run_migrations().await?; - Ok(Arc::new(backend)) + Arc::new(backend) } #[cfg(feature = "postgres")] _ => { @@ -83,13 +86,33 @@ pub async fn connect_from_config( .await .map_err(|e| DatabaseError::Pool(e.to_string()))?; pg.run_migrations().await?; - Ok(Arc::new(pg)) + Arc::new(pg) } #[cfg(not(feature = "postgres"))] - _ => Err(DatabaseError::Pool( - "No database backend available. Enable 'postgres' or 'libsql' feature.".to_string(), - )), + _ => { + return Err(DatabaseError::Pool( + "No database backend available. Enable 'postgres' or 'libsql' feature." + .to_string(), + )); + } + }; + + #[cfg(feature = "lancedb")] + if config.vector_backend == crate::config::VectorBackend::LanceDb { + let path = config + .lancedb_path + .clone() + .unwrap_or_else(crate::config::default_lancedb_path); + let store = crate::workspace::LanceDbVectorStore::new(path) + .await + .map_err(|e| DatabaseError::Pool(format!("LanceDB: {}", e)))?; + return Ok(Arc::new(lancedb_wrapper::DbWithLanceVectorStore::new( + inner, + Arc::new(store), + )) as Arc); } + + Ok(inner) } /// Backend-agnostic database trait. @@ -528,6 +551,9 @@ pub trait Database: Send + Sync { limit: usize, ) -> Result, WorkspaceError>; + /// Get a chunk by ID (for LanceDB update flow). + async fn get_chunk_by_id(&self, chunk_id: Uuid) -> Result, WorkspaceError>; + // ==================== Workspace: Search ==================== /// Perform hybrid search combining FTS and vector similarity. diff --git a/src/db/postgres.rs b/src/db/postgres.rs index 096f3a95..0fa41c50 100644 --- a/src/db/postgres.rs +++ b/src/db/postgres.rs @@ -614,6 +614,13 @@ impl Database for PgBackend { .await } + async fn get_chunk_by_id( + &self, + chunk_id: Uuid, + ) -> Result, WorkspaceError> { + self.repo.get_chunk_by_id(chunk_id).await + } + // ==================== Workspace: Search ==================== async fn hybrid_search( diff --git a/src/main.rs b/src/main.rs index 26d37c49..7ae4c805 100644 --- a/src/main.rs +++ b/src/main.rs @@ -386,7 +386,7 @@ async fn main() -> anyhow::Result<()> { #[cfg(feature = "libsql")] let mut libsql_db: Option> = None; - let db: Option> = if cli.no_db { + let mut inner_db: Option> = if cli.no_db { tracing::warn!("Running without database connection"); None } else { @@ -443,6 +443,42 @@ async fn main() -> anyhow::Result<()> { } }; + // Optionally wrap with LanceDB vector store for workspace semantic search + let db: Option> = if let Some(inner) = inner_db.take() { + #[cfg(feature = "lancedb")] + { + if config.database.vector_backend == ironclaw::config::VectorBackend::LanceDb { + let path = config + .database + .lancedb_path + .clone() + .unwrap_or_else(ironclaw::config::default_lancedb_path); + let store = ironclaw::workspace::LanceDbVectorStore::new(path) + .await + .map_err(|e| anyhow::anyhow!("LanceDB: {}", e))?; + tracing::info!("LanceDB vector store connected for workspace search"); + Some(Arc::new(ironclaw::db::lancedb_wrapper::DbWithLanceVectorStore::new( + inner, + Arc::new(store), + )) as Arc) + } else { + Some(inner) + } + } + #[cfg(not(feature = "lancedb"))] + { + if config.database.vector_backend == ironclaw::config::VectorBackend::LanceDb { + anyhow::bail!( + "VECTOR_BACKEND=lancedb requires the 'lancedb' feature. Build with: cargo build --features lancedb" + ); + } else { + Some(inner) + } + } + } else { + None + }; + // Post-init operations using the database if let Some(ref db) = db { // One-time migration: move disk config files into the DB settings table. diff --git a/src/workspace/lancedb_store.rs b/src/workspace/lancedb_store.rs new file mode 100644 index 00000000..a2b9d70e --- /dev/null +++ b/src/workspace/lancedb_store.rs @@ -0,0 +1,623 @@ +//! LanceDB-backed vector store for workspace memory chunks. +//! +//! Provides an alternative to pgvector/libsql for semantic search when the +//! `lancedb` feature is enabled. Documents and metadata stay in the main +//! database; this store holds chunk embeddings for vector similarity search. +//! +//! Configuration: +//! LANCEDB_PATH=~/.ironclaw/lancedb # Default +//! VECTOR_BACKEND=lancedb # Use LanceDB for vector search + +use std::sync::Arc; + +use async_trait::async_trait; +use uuid::Uuid; + +use crate::error::WorkspaceError; +use crate::workspace::search::RankedResult; + +/// Embedding dimension (text-embedding-3-small default). +/// Must match the embedding model used. +pub const DEFAULT_EMBEDDING_DIM: i32 = 1536; + +/// Vector store abstraction for semantic search. +/// +/// Implementations: pgvector/libsql (embedded in Database), LanceDB (this module). +#[async_trait] +pub trait VectorStore: Send + Sync { + /// Insert a chunk with its embedding. + async fn insert_chunk( + &self, + chunk_id: Uuid, + document_id: Uuid, + user_id: &str, + agent_id: Option, + content: &str, + embedding: &[f32], + ) -> Result<(), WorkspaceError>; + + /// Update an existing chunk's embedding. + /// + /// For LanceDB, this performs delete+insert since LanceDB has limited update + /// support. Caller must provide full chunk metadata for the re-insert. + async fn update_chunk_embedding( + &self, + chunk_id: Uuid, + document_id: Uuid, + user_id: &str, + agent_id: Option, + content: &str, + embedding: &[f32], + ) -> Result<(), WorkspaceError>; + + /// Delete all chunks for a document. + async fn delete_chunks(&self, document_id: Uuid) -> Result<(), WorkspaceError>; + + /// Vector similarity search, filtered by user and agent. + async fn vector_search( + &self, + user_id: &str, + agent_id: Option, + embedding: &[f32], + limit: usize, + ) -> Result, WorkspaceError>; +} + +#[cfg(feature = "lancedb")] +mod impl_lancedb { + use std::sync::Arc; + + use arrow_array::types::Float32Type; + use arrow_array::{ + Array, FixedSizeListArray, RecordBatch, RecordBatchIterator, StringArray, + }; + use arrow_schema::{DataType, Field, Schema}; + use async_trait::async_trait; + use futures::StreamExt; + use lancedb::index::Index; + use uuid::Uuid; + + use super::{RankedResult, VectorStore, DEFAULT_EMBEDDING_DIM}; + use crate::error::WorkspaceError; + + const TABLE_NAME: &str = "memory_chunks"; + + /// LanceDB-backed vector store. + pub struct LanceDbVectorStore { + db: Arc, + table_name: String, + embedding_dim: i32, + } + + impl LanceDbVectorStore { + /// Create a new LanceDB store at the given path. + pub async fn new(path: impl AsRef) -> Result { + let path_str = path + .as_ref() + .to_str() + .ok_or_else(|| WorkspaceError::SearchFailed { + reason: "Invalid LanceDB path".to_string(), + })?; + + let db = lancedb::connect(path_str) + .execute() + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Failed to connect to LanceDB: {}", e), + })?; + + let store = Self { + db: Arc::new(db), + table_name: TABLE_NAME.to_string(), + embedding_dim: DEFAULT_EMBEDDING_DIM, + }; + + store.ensure_table().await?; + Ok(store) + } + + async fn ensure_table(&self) -> Result<(), WorkspaceError> { + let tables = self.db.table_names().execute().await.map_err(|e| { + WorkspaceError::SearchFailed { + reason: format!("Failed to list tables: {}", e), + } + })?; + + if tables.iter().any(|t| t == &self.table_name) { + return Ok(()); + } + + let schema = Arc::new(self.schema()); + self.db + .create_empty_table(&self.table_name, schema.clone()) + .execute() + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Failed to create table: {}", e), + })?; + + let table = self.db.open_table(&self.table_name).execute().await.map_err(|e| { + WorkspaceError::SearchFailed { + reason: format!("Failed to open table: {}", e), + } + })?; + + table + .create_index(&["vector"], Index::Auto) + .execute() + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Failed to create vector index: {}", e), + })?; + + Ok(()) + } + + fn schema(&self) -> Schema { + Schema::new(vec![ + Field::new("chunk_id", DataType::Utf8, false), + Field::new("document_id", DataType::Utf8, false), + Field::new("user_id", DataType::Utf8, false), + Field::new("agent_id", DataType::Utf8, true), + Field::new("content", DataType::Utf8, false), + Field::new( + "vector", + DataType::FixedSizeList( + Arc::new(Field::new("item", DataType::Float32, true)), + self.embedding_dim, + ), + false, + ), + ]) + } + } + + #[async_trait] + impl VectorStore for LanceDbVectorStore { + async fn insert_chunk( + &self, + chunk_id: Uuid, + document_id: Uuid, + user_id: &str, + agent_id: Option, + content: &str, + embedding: &[f32], + ) -> Result<(), WorkspaceError> { + if embedding.len() != self.embedding_dim as usize { + return Err(WorkspaceError::EmbeddingFailed { + reason: format!( + "Embedding dimension {} does not match expected {}", + embedding.len(), + self.embedding_dim + ), + }); + } + + let table = self.db.open_table(&self.table_name).execute().await.map_err(|e| { + WorkspaceError::SearchFailed { + reason: format!("Failed to open table: {}", e), + } + })?; + + let chunk_ids = StringArray::from(vec![chunk_id.to_string()]); + let document_ids = StringArray::from(vec![document_id.to_string()]); + let user_ids = StringArray::from(vec![user_id]); + let agent_ids = StringArray::from(vec![agent_id.map(|a| a.to_string())]); + let contents = StringArray::from(vec![content]); + let vec_values: Vec> = + embedding.iter().map(|&x| Some(x)).collect(); + let vectors = FixedSizeListArray::from_iter_primitive::( + vec![Some(vec_values)].into_iter(), + self.embedding_dim, + ); + + let batch = RecordBatch::try_new( + Arc::new(self.schema()), + vec![ + Arc::new(chunk_ids), + Arc::new(document_ids), + Arc::new(user_ids), + Arc::new(agent_ids), + Arc::new(contents), + Arc::new(vectors), + ], + ) + .map_err(|e| WorkspaceError::ChunkingFailed { + reason: format!("Failed to create record batch: {}", e), + })?; + + let batches = RecordBatchIterator::new( + vec![Ok(batch)].into_iter(), + Arc::new(self.schema()), + ); + + table + .add(Box::new(batches)) + .execute() + .await + .map_err(|e| WorkspaceError::ChunkingFailed { + reason: format!("Failed to insert chunk: {}", e), + })?; + + Ok(()) + } + + async fn update_chunk_embedding( + &self, + chunk_id: Uuid, + document_id: Uuid, + user_id: &str, + agent_id: Option, + content: &str, + embedding: &[f32], + ) -> Result<(), WorkspaceError> { + let table = self.db.open_table(&self.table_name).execute().await.map_err(|e| { + WorkspaceError::SearchFailed { + reason: format!("Failed to open table: {}", e), + } + })?; + + table + .delete(&format!("chunk_id = '{}'", chunk_id)) + .await + .map_err(|e| WorkspaceError::EmbeddingFailed { + reason: format!("Failed to delete chunk for update: {}", e), + })?; + + self.insert_chunk(chunk_id, document_id, user_id, agent_id, content, embedding) + .await + } + + async fn delete_chunks(&self, document_id: Uuid) -> Result<(), WorkspaceError> { + let table = self.db.open_table(&self.table_name).execute().await.map_err(|e| { + WorkspaceError::SearchFailed { + reason: format!("Failed to open table: {}", e), + } + })?; + + table + .delete(&format!("document_id = '{}'", document_id)) + .await + .map_err(|e| WorkspaceError::ChunkingFailed { + reason: format!("Failed to delete chunks: {}", e), + })?; + + Ok(()) + } + + async fn vector_search( + &self, + user_id: &str, + agent_id: Option, + embedding: &[f32], + limit: usize, + ) -> Result, WorkspaceError> { + let table = self.db.open_table(&self.table_name).execute().await.map_err(|e| { + WorkspaceError::SearchFailed { + reason: format!("Failed to open table: {}", e), + } + })?; + + let filter = if let Some(aid) = agent_id { + format!("user_id = '{}' AND agent_id = '{}'", user_id, aid) + } else { + format!("user_id = '{}' AND agent_id IS NULL", user_id) + }; + + let mut stream = table + .query() + .nearest_to(embedding) + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Invalid query vector: {}", e), + })? + .only_if(&filter) + .limit(limit as u32) + .execute() + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Vector search failed: {}", e), + })?; + + let mut results = Vec::new(); + let mut rank: u32 = 1; + while let Some(batch) = stream.next().await { + let batch = batch.map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Stream error: {}", e), + })?; + + let chunk_id_col = batch + .column_by_name("chunk_id") + .ok_or_else(|| WorkspaceError::SearchFailed { + reason: "chunk_id column missing".to_string(), + })?; + let document_id_col = batch + .column_by_name("document_id") + .ok_or_else(|| WorkspaceError::SearchFailed { + reason: "document_id column missing".to_string(), + })?; + let content_col = batch + .column_by_name("content") + .ok_or_else(|| WorkspaceError::SearchFailed { + reason: "content column missing".to_string(), + })?; + + let chunk_ids = chunk_id_col.as_any().downcast_ref::().ok_or_else(|| { + WorkspaceError::SearchFailed { + reason: "chunk_id wrong type".to_string(), + } + })?; + let document_ids = document_id_col.as_any().downcast_ref::().ok_or_else(|| { + WorkspaceError::SearchFailed { + reason: "document_id wrong type".to_string(), + } + })?; + let contents = content_col.as_any().downcast_ref::().ok_or_else(|| { + WorkspaceError::SearchFailed { + reason: "content wrong type".to_string(), + } + })?; + + for i in 0..batch.num_rows() { + let chunk_id = chunk_ids + .value(i) + .parse() + .unwrap_or_else(|_| Uuid::nil()); + let document_id = document_ids + .value(i) + .parse() + .unwrap_or_else(|_| Uuid::nil()); + let content = contents.value(i).to_string(); + + results.push(RankedResult { + chunk_id, + document_id, + content, + rank, + }); + rank += 1; + } + } + + Ok(results) + } + } +} + +#[cfg(feature = "lancedb")] +pub use impl_lancedb::LanceDbVectorStore; + +#[cfg(all(test, feature = "lancedb"))] +mod tests { + use std::sync::Arc; + + use tempfile::TempDir; + use uuid::Uuid; + + use super::{LanceDbVectorStore, VectorStore, DEFAULT_EMBEDDING_DIM}; + + fn make_embedding(seed: f32) -> Vec { + (0..DEFAULT_EMBEDDING_DIM as usize) + .map(|i| (seed * (i as f32 + 1.0)).sin()) + .collect() + } + + #[tokio::test] + async fn test_insert_and_vector_search() { + let dir = TempDir::new().unwrap(); + let store = LanceDbVectorStore::new(dir.path()).await.unwrap(); + + let chunk_id = Uuid::new_v4(); + let document_id = Uuid::new_v4(); + let user_id = "user1"; + let content = "Rust is a systems programming language"; + let embedding = make_embedding(1.0); + + store + .insert_chunk( + chunk_id, + document_id, + user_id, + None, + content, + &embedding, + ) + .await + .unwrap(); + + let results = store + .vector_search(user_id, None, &embedding, 5) + .await + .unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].chunk_id, chunk_id); + assert_eq!(results[0].document_id, document_id); + assert_eq!(results[0].content, content); + assert_eq!(results[0].rank, 1); + } + + #[tokio::test] + async fn test_insert_multiple_and_search_returns_ordered() { + let dir = TempDir::new().unwrap(); + let store = LanceDbVectorStore::new(dir.path()).await.unwrap(); + + let doc_id = Uuid::new_v4(); + let user_id = "user1"; + + // Insert 3 chunks with different embeddings + for (i, seed) in [1.0, 2.0, 3.0].iter().enumerate() { + store + .insert_chunk( + Uuid::new_v4(), + doc_id, + user_id, + None, + &format!("content {}", i), + &make_embedding(*seed), + ) + .await + .unwrap(); + } + + // Search returns all 3, ordered by similarity + let query_emb = make_embedding(2.0); + let results = store + .vector_search(user_id, None, &query_emb, 5) + .await + .unwrap(); + + assert_eq!(results.len(), 3); + let contents: Vec<_> = results.iter().map(|r| r.content.as_str()).collect(); + assert!(contents.contains(&"content 0")); + assert!(contents.contains(&"content 1")); + assert!(contents.contains(&"content 2")); + } + + #[tokio::test] + async fn test_delete_chunks() { + let dir = TempDir::new().unwrap(); + let store = LanceDbVectorStore::new(dir.path()).await.unwrap(); + + let doc_id = Uuid::new_v4(); + let user_id = "user1"; + + store + .insert_chunk( + Uuid::new_v4(), + doc_id, + user_id, + None, + "content", + &make_embedding(1.0), + ) + .await + .unwrap(); + + let results = store + .vector_search(user_id, None, &make_embedding(1.0), 5) + .await + .unwrap(); + assert_eq!(results.len(), 1); + + store.delete_chunks(doc_id).await.unwrap(); + + let results_after = store + .vector_search(user_id, None, &make_embedding(1.0), 5) + .await + .unwrap(); + assert!(results_after.is_empty()); + } + + #[tokio::test] + async fn test_update_chunk_embedding() { + let dir = TempDir::new().unwrap(); + let store = LanceDbVectorStore::new(dir.path()).await.unwrap(); + + let chunk_id = Uuid::new_v4(); + let doc_id = Uuid::new_v4(); + let user_id = "user1"; + let content = "original content"; + + store + .insert_chunk( + chunk_id, + doc_id, + user_id, + None, + content, + &make_embedding(1.0), + ) + .await + .unwrap(); + + // Update with new embedding + let new_embedding = make_embedding(5.0); + store + .update_chunk_embedding(chunk_id, doc_id, user_id, None, content, &new_embedding) + .await + .unwrap(); + + // Search with new embedding should find it + let results = store + .vector_search(user_id, None, &new_embedding, 5) + .await + .unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].chunk_id, chunk_id); + } + + #[tokio::test] + async fn test_vector_search_filters_by_user_and_agent() { + let dir = TempDir::new().unwrap(); + let store = LanceDbVectorStore::new(dir.path()).await.unwrap(); + + let doc_id = Uuid::new_v4(); + let embedding = make_embedding(1.0); + + store + .insert_chunk( + Uuid::new_v4(), + doc_id, + "user1", + None, + "user1 content", + &embedding, + ) + .await + .unwrap(); + + store + .insert_chunk( + Uuid::new_v4(), + doc_id, + "user2", + None, + "user2 content", + &embedding, + ) + .await + .unwrap(); + + let results_user1 = store + .vector_search("user1", None, &embedding, 5) + .await + .unwrap(); + assert_eq!(results_user1.len(), 1); + assert_eq!(results_user1[0].content, "user1 content"); + + let results_user2 = store + .vector_search("user2", None, &embedding, 5) + .await + .unwrap(); + assert_eq!(results_user2.len(), 1); + assert_eq!(results_user2[0].content, "user2 content"); + + let results_wrong_user = store + .vector_search("user3", None, &embedding, 5) + .await + .unwrap(); + assert!(results_wrong_user.is_empty()); + } + + #[tokio::test] + async fn test_insert_rejects_wrong_embedding_dim() { + let dir = TempDir::new().unwrap(); + let store = LanceDbVectorStore::new(dir.path()).await.unwrap(); + + let wrong_dim: Vec = vec![1.0; 100]; + + let err = store + .insert_chunk( + Uuid::new_v4(), + Uuid::new_v4(), + "user1", + None, + "content", + &wrong_dim, + ) + .await + .unwrap_err(); + + assert!(matches!(err, crate::error::WorkspaceError::EmbeddingFailed { .. })); + } +} diff --git a/src/workspace/mod.rs b/src/workspace/mod.rs index 3165ecce..11739212 100644 --- a/src/workspace/mod.rs +++ b/src/workspace/mod.rs @@ -44,6 +44,8 @@ mod chunker; mod document; mod embeddings; pub mod hygiene; +#[cfg(feature = "lancedb")] +pub mod lancedb_store; #[cfg(feature = "postgres")] mod repository; mod search; @@ -53,6 +55,8 @@ pub use document::{MemoryChunk, MemoryDocument, WorkspaceEntry, paths}; pub use embeddings::{EmbeddingProvider, MockEmbeddings, NearAiEmbeddings, OpenAiEmbeddings}; #[cfg(feature = "postgres")] pub use repository::Repository; +#[cfg(feature = "lancedb")] +pub use lancedb_store::{LanceDbVectorStore, VectorStore, DEFAULT_EMBEDDING_DIM}; pub use search::{RankedResult, SearchConfig, SearchResult, reciprocal_rank_fusion}; use std::sync::Arc; diff --git a/src/workspace/repository.rs b/src/workspace/repository.rs index f9e87219..0d05f0d0 100644 --- a/src/workspace/repository.rs +++ b/src/workspace/repository.rs @@ -347,6 +347,36 @@ impl Repository { Ok(()) } + /// Get a chunk by ID. + pub async fn get_chunk_by_id( + &self, + chunk_id: Uuid, + ) -> Result, WorkspaceError> { + let conn = self.conn().await?; + + let row = conn + .query_opt( + r#" + SELECT id, document_id, chunk_index, content, created_at + FROM memory_chunks WHERE id = $1 + "#, + &[&chunk_id], + ) + .await + .map_err(|e| WorkspaceError::SearchFailed { + reason: format!("Query failed: {}", e), + })?; + + Ok(row.map(|r| MemoryChunk { + id: r.get("id"), + document_id: r.get("document_id"), + chunk_index: r.get("chunk_index"), + content: r.get("content"), + embedding: None, + created_at: r.get("created_at"), + })) + } + /// Get chunks without embeddings for backfilling. pub async fn get_chunks_without_embeddings( &self, diff --git a/tests/lancedb_integration.rs b/tests/lancedb_integration.rs new file mode 100644 index 00000000..f92427f2 --- /dev/null +++ b/tests/lancedb_integration.rs @@ -0,0 +1,159 @@ +//! Integration tests for LanceDB vector store with Database wrapper. +//! +//! Requires: cargo test --features "libsql,lancedb" +//! +//! Verifies DbWithLanceVectorStore: document + chunk insert, hybrid search +//! (FTS from libSQL, vector from LanceDB), delete_chunks sync. + +#![cfg(all(feature = "libsql", feature = "lancedb"))] + +use std::sync::Arc; + +use ironclaw::db::lancedb_wrapper::DbWithLanceVectorStore; +use ironclaw::db::Database; +use ironclaw::db::libsql_backend::LibSqlBackend; +use ironclaw::workspace::{LanceDbVectorStore, SearchConfig}; +use tempfile::TempDir; +use uuid::Uuid; + +const EMBEDDING_DIM: usize = 1536; + +fn make_embedding(seed: f32) -> Vec { + (0..EMBEDDING_DIM) + .map(|i| (seed * (i as f32 + 1.0)).sin()) + .collect() +} + +async fn setup_wrapped_db() -> (Arc, TempDir) { + let libsql = LibSqlBackend::new_memory().await.unwrap(); + libsql.run_migrations().await.unwrap(); + + let lancedb_dir = TempDir::new().unwrap(); + let store = LanceDbVectorStore::new(lancedb_dir.path()) + .await + .unwrap(); + + let db = Arc::new(DbWithLanceVectorStore::new( + Arc::new(libsql) as Arc, + Arc::new(store), + )) as Arc; + + (db, lancedb_dir) +} + +#[tokio::test] +async fn test_wrapper_hybrid_search_combines_fts_and_vector() { + let (db, _) = setup_wrapped_db().await; + + let user_id = "test_user"; + let agent_id: Option = None; + + // Create document + let doc = db + .get_or_create_document_by_path(user_id, agent_id, "context/rust.md") + .await + .unwrap(); + + // Write content for FTS + db.update_document(doc.id, "Rust is a systems programming language focused on safety and performance.") + .await + .unwrap(); + + // Chunk and insert with embedding (triggers sync to LanceDB) + let content = "Rust is a systems programming language focused on safety."; + let embedding = make_embedding(1.0); + + let chunk_id = db + .insert_chunk(doc.id, 0, content, Some(&embedding)) + .await + .unwrap(); + + // Hybrid search: FTS for "Rust" + vector for semantic + let config = SearchConfig::default().with_limit(5); + let results = db + .hybrid_search( + user_id, + agent_id, + "Rust", + Some(&embedding), + &config, + ) + .await + .unwrap(); + + assert!(!results.is_empty(), "hybrid search should return results"); + assert_eq!(results[0].chunk_id, chunk_id); + assert!(results[0].content.contains("Rust")); +} + +#[tokio::test] +async fn test_wrapper_delete_chunks_removes_from_both() { + let (db, _) = setup_wrapped_db().await; + + let user_id = "test_user"; + let agent_id: Option = None; + + let doc = db + .get_or_create_document_by_path(user_id, agent_id, "notes/deleted.md") + .await + .unwrap(); + + db.update_document(doc.id, "Content to be deleted.").await.unwrap(); + + let embedding = make_embedding(2.0); + db.insert_chunk(doc.id, 0, "Content to be deleted.", Some(&embedding)) + .await + .unwrap(); + + let before = db + .hybrid_search(user_id, agent_id, "deleted", Some(&embedding), &SearchConfig::default()) + .await + .unwrap(); + assert_eq!(before.len(), 1); + + db.delete_chunks(doc.id).await.unwrap(); + + let after = db + .hybrid_search(user_id, agent_id, "deleted", Some(&embedding), &SearchConfig::default()) + .await + .unwrap(); + assert!(after.is_empty()); +} + +#[tokio::test] +async fn test_wrapper_insert_chunk_syncs_to_lancedb() { + let (db, _) = setup_wrapped_db().await; + + let user_id = "sync_user"; + let agent_id: Option = None; + + let doc = db + .get_or_create_document_by_path(user_id, agent_id, "sync/test.md") + .await + .unwrap(); + + let content = "Semantic content for vector search"; + let embedding = make_embedding(3.0); + + let chunk_id = db + .insert_chunk(doc.id, 0, content, Some(&embedding)) + .await + .unwrap(); + + // Vector-only search (no FTS query match) - should still find via LanceDB + let config = SearchConfig::default().vector_only().with_limit(5); + let results = db + .hybrid_search( + user_id, + agent_id, + "nonexistent_fts_term", + Some(&embedding), + &config, + ) + .await + .unwrap(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].chunk_id, chunk_id); + assert_eq!(results[0].content, content); +}