diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0be3140a..0a414969 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -214,14 +214,113 @@ jobs: path: | ${{ steps.cargo-dist.outputs.paths }} ${{ env.BUILD_MANIFEST_NAME }} + # Build WASM extension bundles (tar.gz with .wasm + .capabilities.json) + build-wasm-extensions: + needs: + - plan + if: ${{ needs.plan.outputs.publishing == 'true' }} + runs-on: "ubuntu-22.04" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: recursive + - name: Install Rust toolchain + wasm target + run: | + rustup target add wasm32-wasip2 + cargo install cargo-component --locked || true + - uses: swatinem/rust-cache@v2 + with: + key: wasm-extensions + - name: Build and package WASM extensions + shell: bash + run: | + set -euo pipefail + mkdir -p target/wasm-bundles + + # Process each manifest in registry/tools/ and registry/channels/ + for manifest in registry/tools/*.json registry/channels/*.json; do + [ -f "$manifest" ] || continue + + name=$(jq -r '.name' "$manifest") + source_dir=$(jq -r '.source.dir' "$manifest") + caps_file=$(jq -r '.source.capabilities' "$manifest") + crate_name=$(jq -r '.source.crate_name' "$manifest") + + if [ ! -d "$source_dir" ]; then + echo "::warning::Source dir '$source_dir' not found for '$name', skipping" + continue + fi + + echo "=== Building $name from $source_dir ===" + + # Build WASM component + cargo component build --release --manifest-path "$source_dir/Cargo.toml" || { + echo "::warning::Build failed for '$name', skipping" + continue + } + + # Find the built WASM file (Cargo uses underscores in artifact names) + wasm_artifact="${crate_name//-/_}" + wasm_path="" + for target_dir in wasm32-wasip2 wasm32-wasip1 wasm32-wasi; do + candidate="$source_dir/target/$target_dir/release/${wasm_artifact}.wasm" + if [ -f "$candidate" ]; then + wasm_path="$candidate" + break + fi + done + + if [ -z "$wasm_path" ]; then + echo "::warning::No WASM output found for '$name', skipping" + continue + fi + + # Copy files with standardized names for the archive + cp "$wasm_path" "target/wasm-bundles/${name}.wasm" + + caps_path="$source_dir/$caps_file" + if [ -f "$caps_path" ]; then + cp "$caps_path" "target/wasm-bundles/${name}.capabilities.json" + else + echo "::warning::No capabilities file at '$caps_path' for '$name'" + fi + + # Create tar.gz bundle + bundle="target/wasm-bundles/${name}-wasm32-wasip2.tar.gz" + (cd target/wasm-bundles && if [ -f "${name}.capabilities.json" ]; then tar czf "${name}-wasm32-wasip2.tar.gz" "${name}.wasm" "${name}.capabilities.json"; else tar czf "${name}-wasm32-wasip2.tar.gz" "${name}.wasm"; fi) + + # Compute SHA256 + sha256=$(sha256sum "$bundle" | cut -d' ' -f1) + echo "$sha256 ${name}-wasm32-wasip2.tar.gz" >> target/wasm-bundles/checksums.txt + + # Clean up intermediate files + rm -f "target/wasm-bundles/${name}.wasm" "target/wasm-bundles/${name}.capabilities.json" + + echo " -> $bundle ($sha256)" + done + + echo "=== WASM bundles built ===" + ls -la target/wasm-bundles/ + - name: "Upload WASM bundles" + uses: actions/upload-artifact@v4 + with: + name: artifacts-wasm-extensions + path: | + target/wasm-bundles/*.tar.gz + target/wasm-bundles/checksums.txt + # Determines if we should publish/announce host: needs: - plan - build-local-artifacts - build-global-artifacts - # Only run if we're "publishing", and only if plan, local and global didn't fail (skipped is fine) - if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') }} + - build-wasm-extensions + # Only run if we're "publishing", and only if plan, local, global, and wasm didn't fail (skipped is fine) + if: ${{ always() && needs.plan.result == 'success' && needs.plan.outputs.publishing == 'true' && (needs.build-global-artifacts.result == 'skipped' || needs.build-global-artifacts.result == 'success') && (needs.build-local-artifacts.result == 'skipped' || needs.build-local-artifacts.result == 'success') && (needs.build-wasm-extensions.result == 'skipped' || needs.build-wasm-extensions.result == 'success') }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} runs-on: "ubuntu-22.04" diff --git a/CLAUDE.md b/CLAUDE.md index cd3c4352..b56cbd5b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,7 +32,7 @@ # Format code cargo fmt -# Lint (address warnings before committing) +# Lint (fix ALL warnings before committing, including pre-existing ones) cargo clippy --all --benches --tests --examples --all-features # Run all tests @@ -321,7 +321,10 @@ cargo check --all-features # all features ``` Dead code behind the wrong `#[cfg]` gate will only show up when building with a single feature. +**Zero clippy warnings policy:** Fix ALL clippy warnings before committing, including pre-existing ones in files you didn't change. Never leave warnings behind — treat `cargo clippy` output as a zero-tolerance gate. + **Mechanical verification before committing:** Run these checks on changed files before committing: +- `cargo clippy --all --benches --tests --examples --all-features` -- zero warnings - `grep -rnE '\.unwrap\(|\.expect\(' ` -- no panics in production - `grep -rn 'super::' ` -- use `crate::` imports - If you fixed a pattern bug, `grep` for other instances of that pattern across `src/` diff --git a/Cargo.lock b/Cargo.lock index 9f4fb8ec..ab4308f0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,6 +11,12 @@ dependencies = [ "gimli", ] +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + [[package]] name = "aead" version = "0.5.2" @@ -1680,6 +1686,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -2508,6 +2524,7 @@ dependencies = [ "deadpool-postgres", "dirs 6.0.0", "dotenvy", + "flate2", "fs4", "futures", "hkdf", @@ -2536,6 +2553,7 @@ dependencies = [ "serde_yml", "sha2", "subtle", + "tar", "tempfile", "termimad", "testcontainers-modules", @@ -2990,6 +3008,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.1.1" @@ -4695,6 +4723,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + [[package]] name = "simdutf8" version = "0.1.5" @@ -4903,6 +4937,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tar" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.16" diff --git a/Cargo.toml b/Cargo.toml index 80f07f51..8bf5c983 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -137,6 +137,10 @@ rig-core = "0.30" # Docker sandbox bollard = "0.18" +# Archive extraction for WASM extension bundles +flate2 = "1" +tar = "0.4" + # HTTP proxy for sandboxed network access hyper = { version = "1.5", features = ["server", "http1", "http2"] } hyper-util = { version = "0.1", features = ["server", "tokio", "http1", "http2"] } diff --git a/build.rs b/build.rs index 8f695ee0..1f644aaf 100644 --- a/build.rs +++ b/build.rs @@ -10,12 +10,17 @@ //! Prerequisites: rustup target add wasm32-wasip2, cargo install wasm-tools use std::env; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::process::Command; fn main() { let manifest_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); let root = PathBuf::from(&manifest_dir); + + // ── Embed registry manifests ──────────────────────────────────────── + embed_registry_catalog(&root); + + // ── Build Telegram channel WASM ───────────────────────────────────── let channel_dir = root.join("channels-src/telegram"); let wasm_out = channel_dir.join("telegram.wasm"); @@ -104,3 +109,89 @@ fn main() { } } } + +/// Collect all registry manifests into a single JSON blob at compile time. +/// +/// Output: `$OUT_DIR/embedded_catalog.json` with structure: +/// ```json +/// { "tools": [...], "channels": [...], "bundles": {...} } +/// ``` +fn embed_registry_catalog(root: &Path) { + use std::fs; + + let registry_dir = root.join("registry"); + + // Rerun if the bundles file changes (per-file watches for tools/channels + // are emitted inside collect_json_files to track content changes reliably). + println!("cargo:rerun-if-changed=registry/_bundles.json"); + + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let out_path = out_dir.join("embedded_catalog.json"); + + if !registry_dir.is_dir() { + // No registry dir: write empty catalog + fs::write( + &out_path, + r#"{"tools":[],"channels":[],"bundles":{"bundles":{}}}"#, + ) + .unwrap(); + return; + } + + let mut tools = Vec::new(); + let mut channels = Vec::new(); + + // Collect tool manifests + let tools_dir = registry_dir.join("tools"); + if tools_dir.is_dir() { + collect_json_files(&tools_dir, &mut tools); + } + + // Collect channel manifests + let channels_dir = registry_dir.join("channels"); + if channels_dir.is_dir() { + collect_json_files(&channels_dir, &mut channels); + } + + // Read bundles + let bundles_path = registry_dir.join("_bundles.json"); + let bundles_raw = if bundles_path.is_file() { + fs::read_to_string(&bundles_path).unwrap_or_else(|_| r#"{"bundles":{}}"#.to_string()) + } else { + r#"{"bundles":{}}"#.to_string() + }; + + // Build the combined JSON + let catalog = format!( + r#"{{"tools":[{}],"channels":[{}],"bundles":{}}}"#, + tools.join(","), + channels.join(","), + bundles_raw, + ); + + fs::write(&out_path, catalog).unwrap(); +} + +/// Read all .json files from a directory and push their raw contents into `out`. +fn collect_json_files(dir: &Path, out: &mut Vec) { + use std::fs; + + let mut entries: Vec<_> = fs::read_dir(dir) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| { + e.path().is_file() && e.path().extension().and_then(|x| x.to_str()) == Some("json") + }) + .collect(); + + // Sort for deterministic output + entries.sort_by_key(|e| e.file_name()); + + for entry in entries { + // Emit per-file watch so Cargo reruns when file contents change + println!("cargo:rerun-if-changed={}", entry.path().display()); + if let Ok(content) = fs::read_to_string(entry.path()) { + out.push(content); + } + } +} diff --git a/registry/channels/discord.json b/registry/channels/discord.json index 77aba7dc..403c8802 100644 --- a/registry/channels/discord.json +++ b/registry/channels/discord.json @@ -14,7 +14,7 @@ "artifacts": { "wasm32-wasip2": { - "url": null, + "url": "https://github.com/nearai/ironclaw/releases/latest/download/discord-wasm32-wasip2.tar.gz", "sha256": null } }, diff --git a/registry/channels/slack.json b/registry/channels/slack.json index bd4c85bf..319cf07e 100644 --- a/registry/channels/slack.json +++ b/registry/channels/slack.json @@ -14,7 +14,7 @@ "artifacts": { "wasm32-wasip2": { - "url": null, + "url": "https://github.com/nearai/ironclaw/releases/latest/download/slack-wasm32-wasip2.tar.gz", "sha256": null } }, diff --git a/registry/channels/telegram.json b/registry/channels/telegram.json index 65a199d9..5c318f01 100644 --- a/registry/channels/telegram.json +++ b/registry/channels/telegram.json @@ -14,7 +14,7 @@ "artifacts": { "wasm32-wasip2": { - "url": null, + "url": "https://github.com/nearai/ironclaw/releases/latest/download/telegram-wasm32-wasip2.tar.gz", "sha256": null } }, diff --git a/registry/channels/whatsapp.json b/registry/channels/whatsapp.json index a36d1e69..56aacced 100644 --- a/registry/channels/whatsapp.json +++ b/registry/channels/whatsapp.json @@ -14,7 +14,7 @@ "artifacts": { "wasm32-wasip2": { - "url": null, + "url": "https://github.com/nearai/ironclaw/releases/latest/download/whatsapp-wasm32-wasip2.tar.gz", "sha256": null } }, diff --git a/registry/tools/github.json b/registry/tools/github.json index 85ee06f7..f1705c73 100644 --- a/registry/tools/github.json +++ b/registry/tools/github.json @@ -14,7 +14,7 @@ "artifacts": { "wasm32-wasip2": { - "url": null, + "url": "https://github.com/nearai/ironclaw/releases/latest/download/github-wasm32-wasip2.tar.gz", "sha256": null } }, diff --git a/registry/tools/gmail.json b/registry/tools/gmail.json index 04c6fd9e..d53c9759 100644 --- a/registry/tools/gmail.json +++ b/registry/tools/gmail.json @@ -14,7 +14,7 @@ "artifacts": { "wasm32-wasip2": { - "url": null, + "url": "https://github.com/nearai/ironclaw/releases/latest/download/gmail-wasm32-wasip2.tar.gz", "sha256": null } }, diff --git a/registry/tools/google-calendar.json b/registry/tools/google-calendar.json index 16d8e89d..477b4b73 100644 --- a/registry/tools/google-calendar.json +++ b/registry/tools/google-calendar.json @@ -14,7 +14,7 @@ "artifacts": { "wasm32-wasip2": { - "url": null, + "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-calendar-wasm32-wasip2.tar.gz", "sha256": null } }, diff --git a/registry/tools/google-docs.json b/registry/tools/google-docs.json index 90e6859a..d60ca2e3 100644 --- a/registry/tools/google-docs.json +++ b/registry/tools/google-docs.json @@ -14,7 +14,7 @@ "artifacts": { "wasm32-wasip2": { - "url": null, + "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-docs-wasm32-wasip2.tar.gz", "sha256": null } }, diff --git a/registry/tools/google-drive.json b/registry/tools/google-drive.json index 586c6afd..a468e48c 100644 --- a/registry/tools/google-drive.json +++ b/registry/tools/google-drive.json @@ -14,7 +14,7 @@ "artifacts": { "wasm32-wasip2": { - "url": null, + "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-drive-wasm32-wasip2.tar.gz", "sha256": null } }, diff --git a/registry/tools/google-sheets.json b/registry/tools/google-sheets.json index f840b6a6..5edddc85 100644 --- a/registry/tools/google-sheets.json +++ b/registry/tools/google-sheets.json @@ -14,7 +14,7 @@ "artifacts": { "wasm32-wasip2": { - "url": null, + "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-sheets-wasm32-wasip2.tar.gz", "sha256": null } }, diff --git a/registry/tools/google-slides.json b/registry/tools/google-slides.json index 94ed4a4a..beb53ff9 100644 --- a/registry/tools/google-slides.json +++ b/registry/tools/google-slides.json @@ -14,7 +14,7 @@ "artifacts": { "wasm32-wasip2": { - "url": null, + "url": "https://github.com/nearai/ironclaw/releases/latest/download/google-slides-wasm32-wasip2.tar.gz", "sha256": null } }, diff --git a/registry/tools/okta.json b/registry/tools/okta.json index 2b55571a..26d17675 100644 --- a/registry/tools/okta.json +++ b/registry/tools/okta.json @@ -14,7 +14,7 @@ "artifacts": { "wasm32-wasip2": { - "url": null, + "url": "https://github.com/nearai/ironclaw/releases/latest/download/okta-wasm32-wasip2.tar.gz", "sha256": null } }, diff --git a/registry/tools/slack.json b/registry/tools/slack.json index 0f876cf3..b7bedf53 100644 --- a/registry/tools/slack.json +++ b/registry/tools/slack.json @@ -14,7 +14,7 @@ "artifacts": { "wasm32-wasip2": { - "url": null, + "url": "https://github.com/nearai/ironclaw/releases/latest/download/slack-wasm32-wasip2.tar.gz", "sha256": null } }, diff --git a/registry/tools/telegram.json b/registry/tools/telegram.json index cd4835c2..d7df228e 100644 --- a/registry/tools/telegram.json +++ b/registry/tools/telegram.json @@ -14,7 +14,7 @@ "artifacts": { "wasm32-wasip2": { - "url": null, + "url": "https://github.com/nearai/ironclaw/releases/latest/download/telegram-wasm32-wasip2.tar.gz", "sha256": null } }, diff --git a/src/app.rs b/src/app.rs index 48407eaf..295d5645 100644 --- a/src/app.rs +++ b/src/app.rs @@ -579,11 +579,44 @@ impl AppBuilder { tokio::join!(wasm_tools_future, mcp_servers_future); - // Create extension manager - let extension_manager = if let Some(ref secrets) = self.secrets_store { + // Load registry catalog entries for extension discovery + let catalog_entries = match crate::registry::RegistryCatalog::load_or_embedded() { + Ok(catalog) => { + let entries: Vec<_> = catalog + .all() + .iter() + .map(|m| m.to_registry_entry()) + .collect(); + tracing::info!( + count = entries.len(), + "Loaded registry catalog entries for extension discovery" + ); + entries + } + Err(e) => { + tracing::warn!("Failed to load registry catalog: {}", e); + Vec::new() + } + }; + + // Create extension manager. Use ephemeral in-memory secrets if no + // persistent store is configured (listing/install/activate still work). + let ext_secrets: Arc = if let Some(ref s) = + self.secrets_store + { + Arc::clone(s) + } else { + use crate::secrets::{InMemorySecretsStore, SecretsCrypto}; + let ephemeral_key = + secrecy::SecretString::from(crate::secrets::keychain::generate_master_key_hex()); + let crypto = Arc::new(SecretsCrypto::new(ephemeral_key).expect("ephemeral crypto")); + tracing::debug!("Using ephemeral in-memory secrets store for extension manager"); + Arc::new(InMemorySecretsStore::new(crypto)) + }; + let extension_manager = { let manager = Arc::new(ExtensionManager::new( Arc::clone(&mcp_session_manager), - Arc::clone(secrets), + ext_secrets, Arc::clone(tools), Some(Arc::clone(hooks)), wasm_tool_runtime.clone(), @@ -592,16 +625,11 @@ impl AppBuilder { self.config.tunnel.public_url.clone(), "default".to_string(), self.db.clone(), + catalog_entries.clone(), )); tools.register_extension_tools(Arc::clone(&manager)); tracing::info!("Extension manager initialized with in-chat discovery tools"); Some(manager) - } else { - tracing::debug!( - "Extension manager not available (no secrets store). \ - Extension tools won't be registered." - ); - None }; // register_builder_tool() already calls register_dev_tools() internally, diff --git a/src/channels/web/mod.rs b/src/channels/web/mod.rs index a212978e..9248e9a1 100644 --- a/src/channels/web/mod.rs +++ b/src/channels/web/mod.rs @@ -89,6 +89,7 @@ impl GatewayChannel { skill_registry: None, skill_catalog: None, chat_rate_limiter: server::RateLimiter::new(30, 60), + registry_entries: Vec::new(), cost_guard: None, startup_time: std::time::Instant::now(), }); @@ -121,6 +122,7 @@ impl GatewayChannel { skill_registry: self.state.skill_registry.clone(), skill_catalog: self.state.skill_catalog.clone(), chat_rate_limiter: server::RateLimiter::new(30, 60), + registry_entries: self.state.registry_entries.clone(), cost_guard: self.state.cost_guard.clone(), startup_time: self.state.startup_time, }; @@ -210,6 +212,12 @@ impl GatewayChannel { self } + /// Inject registry catalog entries for the available extensions API. + pub fn with_registry_entries(mut self, entries: Vec) -> Self { + self.rebuild_state(|s| s.registry_entries = entries); + self + } + /// Inject the cost guard for token/cost tracking in the status popover. pub fn with_cost_guard(mut self, cg: Arc) -> Self { self.rebuild_state(|s| s.cost_guard = Some(cg)); diff --git a/src/channels/web/server.rs b/src/channels/web/server.rs index a4ee00a2..fc8a71f5 100644 --- a/src/channels/web/server.rs +++ b/src/channels/web/server.rs @@ -13,7 +13,7 @@ use axum::{ http::{StatusCode, header}, middleware, response::{ - Html, IntoResponse, + IntoResponse, sse::{Event, KeepAlive, Sse}, }, routing::{get, post}, @@ -148,6 +148,9 @@ pub struct GatewayState { pub skill_catalog: Option>, /// Rate limiter for chat endpoints (30 messages per 60 seconds). pub chat_rate_limiter: RateLimiter, + /// Registry catalog entries for the available extensions API. + /// Populated at startup from `registry/` manifests, independent of extension manager. + pub registry_entries: Vec, /// Cost guard for token/cost tracking. pub cost_guard: Option>, /// Server startup time for uptime calculation. @@ -218,6 +221,7 @@ pub async fn start_server( // Extensions .route("/api/extensions", get(extensions_list_handler)) .route("/api/extensions/tools", get(extensions_tools_handler)) + .route("/api/extensions/registry", get(extensions_registry_handler)) .route("/api/extensions/install", post(extensions_install_handler)) .route( "/api/extensions/{name}/activate", @@ -348,20 +352,32 @@ pub async fn start_server( // --- Static file handlers --- -async fn index_handler() -> Html<&'static str> { - Html(include_str!("static/index.html")) +async fn index_handler() -> impl IntoResponse { + ( + [ + (header::CONTENT_TYPE, "text/html; charset=utf-8"), + (header::CACHE_CONTROL, "no-cache"), + ], + include_str!("static/index.html"), + ) } async fn css_handler() -> impl IntoResponse { ( - [(header::CONTENT_TYPE, "text/css")], + [ + (header::CONTENT_TYPE, "text/css"), + (header::CACHE_CONTROL, "no-cache"), + ], include_str!("static/style.css"), ) } async fn js_handler() -> impl IntoResponse { ( - [(header::CONTENT_TYPE, "application/javascript")], + [ + (header::CONTENT_TYPE, "application/javascript"), + (header::CACHE_CONTROL, "no-cache"), + ], include_str!("static/app.js"), ) } @@ -1722,10 +1738,30 @@ async fn extensions_install_handler( State(state): State>, Json(req): Json, ) -> Result, (StatusCode, String)> { - let ext_mgr = state.extension_manager.as_ref().ok_or(( - StatusCode::NOT_IMPLEMENTED, - "Extension manager not available (secrets store required)".to_string(), - ))?; + // When extension manager isn't available, check registry entries for a helpful message + let Some(ext_mgr) = state.extension_manager.as_ref() else { + // Look up the entry in the catalog to give a specific error + if let Some(entry) = state.registry_entries.iter().find(|e| e.name == req.name) { + let msg = match &entry.source { + crate::extensions::ExtensionSource::WasmBuildable { .. } => { + format!( + "'{}' requires building from source. \ + Run `ironclaw registry install {}` from the CLI.", + req.name, req.name + ) + } + _ => format!( + "Extension manager not available (secrets store required). \ + Configure DATABASE_URL or a secrets backend to enable installation of '{}'.", + req.name + ), + }; + return Ok(Json(ActionResponse::fail(msg))); + } + return Ok(Json(ActionResponse::fail( + "Extension manager not available (secrets store required)".to_string(), + ))); + }; let kind_hint = req.kind.as_deref().and_then(|k| match k { "mcp_server" => Some(crate::extensions::ExtensionKind::McpServer), @@ -1874,6 +1910,68 @@ async fn extensions_remove_handler( } } +async fn extensions_registry_handler( + State(state): State>, + Query(params): Query, +) -> Json { + let query = params.query.unwrap_or_default(); + let query_lower = query.to_lowercase(); + let tokens: Vec<&str> = query_lower.split_whitespace().collect(); + + // Filter registry entries by query (or return all if empty) + let matching: Vec<&crate::extensions::RegistryEntry> = if tokens.is_empty() { + state.registry_entries.iter().collect() + } else { + state + .registry_entries + .iter() + .filter(|e| { + let name = e.name.to_lowercase(); + let display = e.display_name.to_lowercase(); + let desc = e.description.to_lowercase(); + tokens.iter().any(|t| { + name.contains(t) + || display.contains(t) + || desc.contains(t) + || e.keywords.iter().any(|k| k.to_lowercase().contains(t)) + }) + }) + .collect() + }; + + // Cross-reference with installed extensions by (name, kind) to avoid + // false positives when the same name exists as different kinds. + let installed: std::collections::HashSet<(String, String)> = + if let Some(ext_mgr) = state.extension_manager.as_ref() { + ext_mgr + .list(None) + .await + .unwrap_or_default() + .into_iter() + .map(|ext| (ext.name, ext.kind.to_string())) + .collect() + } else { + std::collections::HashSet::new() + }; + + let entries = matching + .into_iter() + .map(|e| { + let kind_str = e.kind.to_string(); + RegistryEntryInfo { + name: e.name.clone(), + display_name: e.display_name.clone(), + installed: installed.contains(&(e.name.clone(), kind_str.clone())), + kind: kind_str, + description: e.description.clone(), + keywords: e.keywords.clone(), + } + }) + .collect(); + + Json(RegistrySearchResponse { entries }) +} + // --- Skills handlers --- async fn skills_list_handler( diff --git a/src/channels/web/types.rs b/src/channels/web/types.rs index c64a8bd0..a015c6f2 100644 --- a/src/channels/web/types.rs +++ b/src/channels/web/types.rs @@ -408,6 +408,28 @@ impl ActionResponse { } } +// --- Registry --- + +#[derive(Debug, Serialize)] +pub struct RegistryEntryInfo { + pub name: String, + pub display_name: String, + pub kind: String, + pub description: String, + pub keywords: Vec, + pub installed: bool, +} + +#[derive(Debug, Serialize)] +pub struct RegistrySearchResponse { + pub entries: Vec, +} + +#[derive(Debug, Deserialize)] +pub struct RegistrySearchQuery { + pub query: Option, +} + // --- Skills --- #[derive(Debug, Serialize)] diff --git a/src/channels/web/ws.rs b/src/channels/web/ws.rs index 6e91717d..96c6f783 100644 --- a/src/channels/web/ws.rs +++ b/src/channels/web/ws.rs @@ -490,6 +490,7 @@ mod tests { skill_registry: None, skill_catalog: None, chat_rate_limiter: crate::channels::web::server::RateLimiter::new(30, 60), + registry_entries: Vec::new(), cost_guard: None, startup_time: std::time::Instant::now(), } diff --git a/src/cli/registry.rs b/src/cli/registry.rs index 76dc77a8..0126db6f 100644 --- a/src/cli/registry.rs +++ b/src/cli/registry.rs @@ -1,7 +1,5 @@ //! Registry CLI commands for discovering and installing extensions. -use std::path::PathBuf; - use clap::Subcommand; use crate::registry::catalog::RegistryCatalog; @@ -59,8 +57,20 @@ pub enum RegistryCommand { /// Run a registry command. pub async fn run_registry_command(cmd: RegistryCommand) -> anyhow::Result<()> { - let registry_dir = find_registry_dir()?; - let catalog = RegistryCatalog::load(®istry_dir)?; + // For install commands that need to build from source, a disk registry is required. + // For list/info, embedded manifests suffice. + let registry_dir = RegistryCatalog::find_dir(); + let catalog = if let Some(ref dir) = registry_dir { + RegistryCatalog::load(dir)? + } else { + RegistryCatalog::load_or_embedded()? + }; + + // Resolve repo root for installer (empty path when running from binary) + let repo_root = registry_dir + .as_ref() + .and_then(|d| d.parent().map(|p| p.to_path_buf())) + .unwrap_or_default(); match cmd { RegistryCommand::List { kind, tag, verbose } => { @@ -68,53 +78,14 @@ pub async fn run_registry_command(cmd: RegistryCommand) -> anyhow::Result<()> { } RegistryCommand::Info { name } => cmd_info(&catalog, &name), RegistryCommand::Install { name, force, build } => { - cmd_install(&catalog, ®istry_dir, &name, force, build).await + cmd_install(&catalog, &repo_root, &name, force, build).await } RegistryCommand::InstallDefaults { force, build } => { - cmd_install(&catalog, ®istry_dir, "default", force, build).await + cmd_install(&catalog, &repo_root, "default", force, build).await } } } -/// Find the registry directory by looking relative to the current executable or cwd. -fn find_registry_dir() -> anyhow::Result { - // Try relative to current directory (for dev usage) - let cwd = std::env::current_dir()?; - let candidate = cwd.join("registry"); - if candidate.is_dir() { - return Ok(candidate); - } - - // Try relative to executable (covers installed binary, target/debug/, target/release/) - if let Ok(exe) = std::env::current_exe() - && let Some(parent) = exe.parent() - { - // Walk up to 3 levels: exe dir, parent (target/release → target), grandparent (→ repo root) - let mut dir = Some(parent); - for _ in 0..3 { - if let Some(d) = dir { - let candidate = d.join("registry"); - if candidate.is_dir() { - return Ok(candidate); - } - dir = d.parent(); - } - } - } - - // Try CARGO_MANIFEST_DIR (compile-time, works in dev builds) - let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); - let candidate = manifest_dir.join("registry"); - if candidate.is_dir() { - return Ok(candidate); - } - - anyhow::bail!( - "Could not find registry/ directory. Run from the ironclaw repo root, \ - or ensure registry/ is next to the ironclaw binary." - ) -} - fn cmd_list( catalog: &RegistryCatalog, kind: Option<&str>, @@ -254,16 +225,11 @@ fn cmd_info(catalog: &RegistryCatalog, name: &str) -> anyhow::Result<()> { async fn cmd_install( catalog: &RegistryCatalog, - registry_dir: &std::path::Path, + repo_root: &std::path::Path, name: &str, force: bool, prefer_build: bool, ) -> anyhow::Result<()> { - // Registry dir parent is the repo root - let repo_root = registry_dir - .parent() - .ok_or_else(|| anyhow::anyhow!("Cannot determine repo root from registry dir"))?; - let installer = RegistryInstaller::with_defaults(repo_root.to_path_buf()); let (manifests, bundle) = catalog.resolve(name)?; diff --git a/src/extensions/manager.rs b/src/extensions/manager.rs index daca5a33..16315e51 100644 --- a/src/extensions/manager.rs +++ b/src/extensions/manager.rs @@ -75,9 +75,15 @@ impl ExtensionManager { tunnel_url: Option, user_id: String, store: Option>, + catalog_entries: Vec, ) -> Self { + let registry = if catalog_entries.is_empty() { + ExtensionRegistry::new() + } else { + ExtensionRegistry::new_with_catalog(catalog_entries) + }; Self { - registry: ExtensionRegistry::new(), + registry, discovery: OnlineDiscovery::new(), mcp_session_manager, mcp_clients: RwLock::new(HashMap::new()), @@ -131,9 +137,14 @@ impl ExtensionManager { url: Option<&str>, kind_hint: Option, ) -> Result { + tracing::info!(extension = %name, url = ?url, kind = ?kind_hint, "Installing extension"); + // If we have a registry entry, use it if let Some(entry) = self.registry.get(name).await { - return self.install_from_entry(&entry).await; + return self.install_from_entry(&entry).await.map_err(|e| { + tracing::error!(extension = %name, error = %e, "Extension install failed"); + e + }); } // If a URL was provided, determine kind and install @@ -143,19 +154,21 @@ impl ExtensionManager { ExtensionKind::McpServer => self.install_mcp_from_url(name, url).await, ExtensionKind::WasmTool => self.install_wasm_tool_from_url(name, url).await, ExtensionKind::WasmChannel => { - Err(ExtensionError::InstallFailed( - "WASM channel installation from URL not yet supported. \ - Place the .wasm and .capabilities.json files in ~/.ironclaw/channels/ and restart." - .to_string(), - )) + self.install_wasm_channel_from_url(name, url, None).await } - }; + } + .map_err(|e| { + tracing::error!(extension = %name, url = %url, error = %e, "Extension install from URL failed"); + e + }); } - Err(ExtensionError::NotFound(format!( + let err = ExtensionError::NotFound(format!( "'{}' not found in registry. Try searching with discover:true or provide a URL.", name - ))) + )); + tracing::warn!(extension = %name, "Extension not found in registry"); + Err(err) } /// Authenticate an installed extension. @@ -433,16 +446,51 @@ impl ExtensionManager { self.install_mcp_from_url(&entry.name, &url).await } ExtensionKind::WasmTool => match &entry.source { - ExtensionSource::WasmDownload { wasm_url, .. } => { - self.install_wasm_tool_from_url(&entry.name, wasm_url).await + ExtensionSource::WasmDownload { + wasm_url, + capabilities_url, + } => { + self.install_wasm_tool_from_url_with_caps( + &entry.name, + wasm_url, + capabilities_url.as_deref(), + ) + .await + } + ExtensionSource::WasmBuildable { .. } => { + Err(ExtensionError::InstallFailed(format!( + "'{}' requires building from source. Run `ironclaw registry install {}` \ + from the CLI (requires cargo-component).", + entry.name, entry.name + ))) } _ => Err(ExtensionError::InstallFailed( "WASM tool entry has no download URL".to_string(), )), }, - ExtensionKind::WasmChannel => Err(ExtensionError::InstallFailed( - "WASM channel installation not yet supported via this flow".to_string(), - )), + ExtensionKind::WasmChannel => match &entry.source { + ExtensionSource::WasmDownload { + wasm_url, + capabilities_url, + } => { + self.install_wasm_channel_from_url( + &entry.name, + wasm_url, + capabilities_url.as_deref(), + ) + .await + } + ExtensionSource::WasmBuildable { .. } => { + Err(ExtensionError::InstallFailed(format!( + "'{}' requires building from source. Run `ironclaw registry install {}` \ + from the CLI (requires cargo-component).", + entry.name, entry.name + ))) + } + _ => Err(ExtensionError::InstallFailed( + "WASM channel entry has no download URL".to_string(), + )), + }, } } @@ -482,6 +530,57 @@ impl ExtensionManager { name: &str, url: &str, ) -> Result { + self.install_wasm_tool_from_url_with_caps(name, url, None) + .await + } + + async fn install_wasm_tool_from_url_with_caps( + &self, + name: &str, + url: &str, + capabilities_url: Option<&str>, + ) -> Result { + self.download_and_install_wasm(name, url, capabilities_url, &self.wasm_tools_dir) + .await?; + + Ok(InstallResult { + name: name.to_string(), + kind: ExtensionKind::WasmTool, + message: format!("WASM tool '{}' installed. Run activate to load it.", name), + }) + } + + async fn install_wasm_channel_from_url( + &self, + name: &str, + url: &str, + capabilities_url: Option<&str>, + ) -> Result { + self.download_and_install_wasm(name, url, capabilities_url, &self.wasm_channels_dir) + .await?; + + Ok(InstallResult { + name: name.to_string(), + kind: ExtensionKind::WasmChannel, + message: format!( + "WASM channel '{}' installed to {}. Restart to activate.", + name, + self.wasm_channels_dir.display() + ), + }) + } + + /// Download a WASM extension (tool or channel) from URL and install to target directory. + /// + /// Handles both tar.gz bundles (containing `.wasm` + `.capabilities.json`) and bare + /// `.wasm` files. Validates HTTPS, size limits, and file format. + async fn download_and_install_wasm( + &self, + name: &str, + url: &str, + capabilities_url: Option<&str>, + target_dir: &std::path::Path, + ) -> Result<(), ExtensionError> { // Require HTTPS to prevent downgrade attacks if !url.starts_with("https://") { return Err(ExtensionError::InstallFailed( @@ -490,33 +589,41 @@ impl ExtensionManager { } // 50 MB cap to prevent disk-fill DoS - const MAX_WASM_SIZE: usize = 50 * 1024 * 1024; + const MAX_DOWNLOAD_SIZE: usize = 50 * 1024 * 1024; let client = reqwest::Client::builder() .timeout(std::time::Duration::from_secs(60)) .build() .map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?; - let response = client - .get(url) - .send() - .await - .map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?; + tracing::debug!(extension = %name, url = %url, "Downloading WASM extension"); + + let response = client.get(url).send().await.map_err(|e| { + tracing::error!(extension = %name, url = %url, error = %e, "Download request failed"); + ExtensionError::DownloadFailed(e.to_string()) + })?; if !response.status().is_success() { + let status = response.status(); + tracing::error!( + extension = %name, + url = %url, + status = %status, + "Download returned non-success HTTP status" + ); return Err(ExtensionError::DownloadFailed(format!( - "HTTP {}", - response.status() + "HTTP {} from {}", + status, url ))); } // Check Content-Length header before downloading the full body if let Some(len) = response.content_length() - && len as usize > MAX_WASM_SIZE + && len as usize > MAX_DOWNLOAD_SIZE { return Err(ExtensionError::InstallFailed(format!( - "WASM binary too large ({} bytes, max {} bytes)", - len, MAX_WASM_SIZE + "Download too large ({} bytes, max {} bytes)", + len, MAX_DOWNLOAD_SIZE ))); } @@ -525,45 +632,164 @@ impl ExtensionManager { .await .map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?; - if bytes.len() > MAX_WASM_SIZE { + if bytes.len() > MAX_DOWNLOAD_SIZE { return Err(ExtensionError::InstallFailed(format!( - "WASM binary too large ({} bytes, max {} bytes)", + "Download too large ({} bytes, max {} bytes)", bytes.len(), - MAX_WASM_SIZE + MAX_DOWNLOAD_SIZE ))); } - // Basic WASM magic number check (\0asm) - if bytes.len() < 4 || &bytes[..4] != b"\0asm" { - return Err(ExtensionError::InstallFailed( - "Downloaded file is not a valid WASM binary (bad magic number)".to_string(), - )); + // Ensure target directory exists + tokio::fs::create_dir_all(target_dir) + .await + .map_err(|e| ExtensionError::InstallFailed(e.to_string()))?; + + let wasm_path = target_dir.join(format!("{}.wasm", name)); + let caps_path = target_dir.join(format!("{}.capabilities.json", name)); + + // Detect format: gzip (tar.gz bundle) or bare WASM + if bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b { + // tar.gz bundle: extract {name}.wasm and {name}.capabilities.json + self.extract_wasm_tar_gz(name, &bytes, &wasm_path, &caps_path)?; + } else { + // Bare WASM file: validate magic number + if bytes.len() < 4 || &bytes[..4] != b"\0asm" { + return Err(ExtensionError::InstallFailed( + "Downloaded file is not a valid WASM binary (bad magic number)".to_string(), + )); + } + + tokio::fs::write(&wasm_path, &bytes) + .await + .map_err(|e| ExtensionError::InstallFailed(e.to_string()))?; + + // Download capabilities separately if URL provided + if let Some(caps_url) = capabilities_url { + const MAX_CAPS_SIZE: usize = 1024 * 1024; // 1 MB + match client.get(caps_url).send().await { + Ok(resp) if resp.status().is_success() => match resp.bytes().await { + Ok(caps_bytes) if caps_bytes.len() <= MAX_CAPS_SIZE => { + if let Err(e) = tokio::fs::write(&caps_path, &caps_bytes).await { + tracing::warn!( + "Failed to write capabilities for '{}': {}", + name, + e + ); + } + } + Ok(caps_bytes) => { + tracing::warn!( + "Capabilities file for '{}' too large ({} bytes, max {})", + name, + caps_bytes.len(), + MAX_CAPS_SIZE + ); + } + Err(e) => { + tracing::warn!("Failed to download capabilities for '{}': {}", name, e); + } + }, + _ => { + tracing::warn!( + "Failed to download capabilities for '{}' from {}", + name, + caps_url + ); + } + } + } } - // Ensure tools directory exists - tokio::fs::create_dir_all(&self.wasm_tools_dir) - .await - .map_err(|e| ExtensionError::InstallFailed(e.to_string()))?; - - // Write the WASM file - let wasm_path = self.wasm_tools_dir.join(format!("{}.wasm", name)); - tokio::fs::write(&wasm_path, &bytes) - .await - .map_err(|e| ExtensionError::InstallFailed(e.to_string()))?; - tracing::info!( - "Installed WASM tool '{}' ({} bytes) from {} to {}", + "Installed WASM extension '{}' from {} to {}", name, - bytes.len(), url, wasm_path.display() ); - Ok(InstallResult { - name: name.to_string(), - kind: ExtensionKind::WasmTool, - message: format!("WASM tool '{}' installed. Run activate to load it.", name), - }) + Ok(()) + } + + /// Extract a tar.gz bundle into the WASM tools directory. + fn extract_wasm_tar_gz( + &self, + name: &str, + bytes: &[u8], + target_wasm: &std::path::Path, + target_caps: &std::path::Path, + ) -> Result<(), ExtensionError> { + use flate2::read::GzDecoder; + use tar::Archive; + + use std::io::Read as _; + + let decoder = GzDecoder::new(bytes); + let mut archive = Archive::new(decoder); + // Defense-in-depth: do not preserve permissions or extended attributes + archive.set_preserve_permissions(false); + #[cfg(any(unix, target_os = "redox"))] + archive.set_unpack_xattrs(false); + + // 100 MB cap on decompressed entry size to prevent decompression bombs + const MAX_ENTRY_SIZE: u64 = 100 * 1024 * 1024; + + let wasm_filename = format!("{}.wasm", name); + let caps_filename = format!("{}.capabilities.json", name); + let mut found_wasm = false; + + let entries = archive + .entries() + .map_err(|e| ExtensionError::InstallFailed(format!("Bad tar.gz archive: {}", e)))?; + + for entry in entries { + let mut entry = entry + .map_err(|e| ExtensionError::InstallFailed(format!("Bad tar.gz entry: {}", e)))?; + + if entry.size() > MAX_ENTRY_SIZE { + return Err(ExtensionError::InstallFailed(format!( + "Archive entry too large ({} bytes, max {} bytes)", + entry.size(), + MAX_ENTRY_SIZE + ))); + } + + let entry_path = entry + .path() + .map_err(|e| { + ExtensionError::InstallFailed(format!("Invalid path in tar.gz: {}", e)) + })? + .to_path_buf(); + + let filename = entry_path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or(""); + + if filename == wasm_filename { + let mut data = Vec::with_capacity(entry.size() as usize); + std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data) + .map_err(|e| ExtensionError::InstallFailed(e.to_string()))?; + std::fs::write(target_wasm, &data) + .map_err(|e| ExtensionError::InstallFailed(e.to_string()))?; + found_wasm = true; + } else if filename == caps_filename { + let mut data = Vec::with_capacity(entry.size() as usize); + std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data) + .map_err(|e| ExtensionError::InstallFailed(e.to_string()))?; + std::fs::write(target_caps, &data) + .map_err(|e| ExtensionError::InstallFailed(e.to_string()))?; + } + } + + if !found_wasm { + return Err(ExtensionError::InstallFailed(format!( + "tar.gz archive does not contain '{}'", + wasm_filename + ))); + } + + Ok(()) } async fn auth_mcp( @@ -1074,7 +1300,7 @@ impl ExtensionManager { /// Infer the extension kind from a URL. fn infer_kind_from_url(url: &str) -> ExtensionKind { - if url.ends_with(".wasm") { + if url.ends_with(".wasm") || url.ends_with(".tar.gz") { ExtensionKind::WasmTool } else { ExtensionKind::McpServer @@ -1092,6 +1318,10 @@ mod tests { infer_kind_from_url("https://example.com/tool.wasm"), ExtensionKind::WasmTool ); + assert_eq!( + infer_kind_from_url("https://example.com/tool-wasm32-wasip2.tar.gz"), + ExtensionKind::WasmTool + ); assert_eq!( infer_kind_from_url("https://mcp.notion.com"), ExtensionKind::McpServer diff --git a/src/extensions/registry.rs b/src/extensions/registry.rs index 8d136b08..bb2cd1c3 100644 --- a/src/extensions/registry.rs +++ b/src/extensions/registry.rs @@ -26,6 +26,26 @@ impl ExtensionRegistry { } } + /// Create a new registry merging builtin entries with catalog-provided entries. + /// + /// Deduplicates by `(name, kind)` pair -- a builtin MCP "slack" and a registry + /// WASM "slack" can coexist since they're different kinds. + pub fn new_with_catalog(catalog_entries: Vec) -> Self { + let mut entries = builtin_entries(); + for entry in catalog_entries { + if !entries + .iter() + .any(|e| e.name == entry.name && e.kind == entry.kind) + { + entries.push(entry); + } + } + Self { + entries, + discovery_cache: RwLock::new(Vec::new()), + } + } + /// Search the registry by query string. Returns results sorted by relevance. /// /// Splits the query into lowercase tokens and scores each entry by matches @@ -542,4 +562,76 @@ mod tests { let results = registry.search("dup").await; assert_eq!(results.len(), 1, "Should not duplicate cached entries"); } + + #[tokio::test] + async fn test_new_with_catalog() { + let catalog_entries = vec![ + RegistryEntry { + name: "telegram".to_string(), + display_name: "Telegram".to_string(), + kind: ExtensionKind::WasmChannel, + description: "Telegram Bot API channel".to_string(), + keywords: vec!["messaging".into(), "bot".into()], + source: ExtensionSource::WasmBuildable { + repo_url: "channels-src/telegram".to_string(), + build_dir: Some("channels-src/telegram".to_string()), + }, + auth_hint: AuthHint::CapabilitiesAuth, + }, + // This shares a name with a builtin but has a different kind, so both should appear + RegistryEntry { + name: "slack".to_string(), + display_name: "Slack WASM".to_string(), + kind: ExtensionKind::WasmTool, + description: "Slack WASM tool".to_string(), + keywords: vec!["messaging".into()], + source: ExtensionSource::WasmBuildable { + repo_url: "tools-src/slack".to_string(), + build_dir: Some("tools-src/slack".to_string()), + }, + auth_hint: AuthHint::CapabilitiesAuth, + }, + ]; + + let registry = ExtensionRegistry::new_with_catalog(catalog_entries); + + // Should find the new telegram entry + let results = registry.search("telegram").await; + assert!(!results.is_empty(), "Should find telegram from catalog"); + assert_eq!(results[0].entry.name, "telegram"); + + // Should have both builtin MCP slack and catalog WASM slack + let results = registry.search("slack").await; + let slack_mcp = results + .iter() + .any(|r| r.entry.name == "slack" && r.entry.kind == ExtensionKind::McpServer); + let slack_wasm = results + .iter() + .any(|r| r.entry.name == "slack" && r.entry.kind == ExtensionKind::WasmTool); + assert!(slack_mcp, "Should have builtin MCP slack"); + assert!(slack_wasm, "Should have catalog WASM slack"); + } + + #[tokio::test] + async fn test_new_with_catalog_dedup_same_kind() { + // A catalog entry with same name AND kind as a builtin should be skipped + let catalog_entries = vec![RegistryEntry { + name: "slack".to_string(), + display_name: "Slack Override".to_string(), + kind: ExtensionKind::McpServer, // same kind as builtin + description: "Should be skipped".to_string(), + keywords: vec![], + source: ExtensionSource::McpUrl { + url: "https://other.slack.com".to_string(), + }, + auth_hint: AuthHint::Dcr, + }]; + + let registry = ExtensionRegistry::new_with_catalog(catalog_entries); + + let entry = registry.get("slack").await; + assert!(entry.is_some()); + // Should still be the builtin, not the override + assert_eq!(entry.unwrap().display_name, "Slack"); + } } diff --git a/src/main.rs b/src/main.rs index f02da188..5d6a20e7 100644 --- a/src/main.rs +++ b/src/main.rs @@ -909,11 +909,43 @@ async fn main() -> anyhow::Result<()> { let (dev_loaded_tool_names, _) = tokio::join!(wasm_tools_future, mcp_servers_future); - // Create extension manager for in-chat discovery/install/auth/activate - let extension_manager = if let Some(ref secrets) = secrets_store { + // Load registry catalog entries for in-chat extension discovery + let catalog_entries = match ironclaw::registry::RegistryCatalog::load_or_embedded() { + Ok(catalog) => { + let entries: Vec = catalog + .all() + .iter() + .map(|m| m.to_registry_entry()) + .collect(); + tracing::info!( + count = entries.len(), + "Loaded registry catalog entries for extension discovery" + ); + entries + } + Err(e) => { + tracing::warn!("Failed to load registry catalog: {}", e); + Vec::new() + } + }; + + // Create extension manager for in-chat discovery/install/auth/activate. + // If no persistent secrets store is available, use an ephemeral in-memory store + // so that listing/installing/activating extensions still works (auth won't persist). + let ext_secrets: Arc = if let Some(ref s) = secrets_store { + Arc::clone(s) + } else { + use ironclaw::secrets::{InMemorySecretsStore, SecretsCrypto}; + let ephemeral_key = + secrecy::SecretString::from(ironclaw::secrets::keychain::generate_master_key_hex()); + let crypto = Arc::new(SecretsCrypto::new(ephemeral_key).expect("ephemeral crypto")); + tracing::debug!("Using ephemeral in-memory secrets store for extension manager"); + Arc::new(InMemorySecretsStore::new(crypto)) + }; + let extension_manager = { let manager = Arc::new(ExtensionManager::new( Arc::clone(&mcp_session_manager), - Arc::clone(secrets), + ext_secrets, Arc::clone(&tools), Some(Arc::clone(&hooks)), wasm_tool_runtime.clone(), @@ -922,16 +954,11 @@ async fn main() -> anyhow::Result<()> { config.tunnel.public_url.clone(), "default".to_string(), db.clone(), + catalog_entries.clone(), )); tools.register_extension_tools(Arc::clone(&manager)); tracing::info!("Extension manager initialized with in-chat discovery tools"); Some(manager) - } else { - tracing::debug!( - "Extension manager not available (no secrets store). \ - Extension tools won't be registered." - ); - None }; // Set up orchestrator for sandboxed job execution @@ -1341,6 +1368,9 @@ async fn main() -> anyhow::Result<()> { if let Some(ref ext_mgr) = extension_manager { gw = gw.with_extension_manager(Arc::clone(ext_mgr)); } + if !catalog_entries.is_empty() { + gw = gw.with_registry_entries(catalog_entries.clone()); + } if let Some(ref d) = db { gw = gw.with_store(Arc::clone(d)); } diff --git a/src/registry/catalog.rs b/src/registry/catalog.rs index 64e8d5a8..7c264aa0 100644 --- a/src/registry/catalog.rs +++ b/src/registry/catalog.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; +use crate::registry::embedded; use crate::registry::manifest::{BundleDefinition, BundlesFile, ExtensionManifest, ManifestKind}; /// Error type for registry operations. @@ -64,6 +65,69 @@ pub struct RegistryCatalog { } impl RegistryCatalog { + /// Find the `registry/` directory by searching relative to cwd, the executable, + /// and `CARGO_MANIFEST_DIR`. Returns `None` if the directory cannot be found + /// (non-fatal at startup). + pub fn find_dir() -> Option { + // Try relative to current directory (for dev usage) + if let Ok(cwd) = std::env::current_dir() { + let candidate = cwd.join("registry"); + if candidate.is_dir() { + return Some(candidate); + } + } + + // Try relative to executable (covers installed binary, target/debug/, target/release/) + if let Ok(exe) = std::env::current_exe() + && let Some(parent) = exe.parent() + { + // Walk up to 3 levels: exe dir, parent (target/release -> target), grandparent (-> repo root) + let mut dir = Some(parent); + for _ in 0..3 { + if let Some(d) = dir { + let candidate = d.join("registry"); + if candidate.is_dir() { + return Some(candidate); + } + dir = d.parent(); + } + } + } + + // Try CARGO_MANIFEST_DIR (compile-time, works in dev builds) + let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); + let candidate = manifest_dir.join("registry"); + if candidate.is_dir() { + return Some(candidate); + } + + None + } + + /// Try to load from disk; if `registry/` cannot be found, fall back to + /// manifests embedded into the binary at compile time. + pub fn load_or_embedded() -> Result { + if let Some(dir) = Self::find_dir() { + return Self::load(&dir); + } + + // Fall back to embedded catalog + let manifests = embedded::load_embedded(); + let bundles = embedded::load_embedded_bundles(); + + tracing::info!( + "Loaded embedded registry catalog ({} extensions, {} bundles)", + manifests.len(), + bundles.len() + ); + + Ok(Self { + manifests, + bundles, + root: PathBuf::new(), + }) + } + /// Load the catalog from a registry directory. /// /// Expects the structure: @@ -577,4 +641,12 @@ mod tests { let result = RegistryCatalog::load(Path::new("/nonexistent/path")); assert!(result.is_err()); } + + #[test] + fn test_load_or_embedded_succeeds() { + // Should always succeed: either finds registry/ on disk or falls back to embedded + let catalog = RegistryCatalog::load_or_embedded().unwrap(); + // At minimum, the embedded catalog from the repo should have entries + assert!(!catalog.all().is_empty() || !catalog.bundle_names().is_empty()); + } } diff --git a/src/registry/embedded.rs b/src/registry/embedded.rs new file mode 100644 index 00000000..4c61ada7 --- /dev/null +++ b/src/registry/embedded.rs @@ -0,0 +1,97 @@ +//! Embedded registry catalog compiled into the binary at build time. +//! +//! When IronClaw is distributed as a pre-built binary without a source tree, +//! the `registry/` directory is unavailable. This module provides the same +//! manifest data via `include_str!` from a JSON blob generated by `build.rs`. + +use std::collections::HashMap; +use std::sync::OnceLock; + +use crate::registry::manifest::{BundleDefinition, BundlesFile, ExtensionManifest}; + +/// Raw JSON generated by build.rs from `registry/{tools,channels}/*.json` and `_bundles.json`. +const EMBEDDED_CATALOG: &str = include_str!(concat!(env!("OUT_DIR"), "/embedded_catalog.json")); + +/// Intermediate deserialization shape matching the build.rs output. +#[derive(serde::Deserialize)] +struct EmbeddedCatalogRaw { + #[serde(default)] + tools: Vec, + #[serde(default)] + channels: Vec, + #[serde(default)] + bundles: BundlesFile, +} + +/// Parsed catalog cached across calls. +struct ParsedCatalog { + manifests: HashMap, + bundles: HashMap, +} + +fn parsed_catalog() -> &'static ParsedCatalog { + static CACHE: OnceLock = OnceLock::new(); + CACHE.get_or_init(|| { + let raw: EmbeddedCatalogRaw = match serde_json::from_str(EMBEDDED_CATALOG) { + Ok(v) => v, + Err(e) => { + tracing::warn!("Failed to parse embedded catalog: {}", e); + return ParsedCatalog { + manifests: HashMap::new(), + bundles: HashMap::new(), + }; + } + }; + + let mut manifests = HashMap::new(); + for m in raw.tools { + let key = format!("tools/{}", m.name); + manifests.insert(key, m); + } + for m in raw.channels { + let key = format!("channels/{}", m.name); + manifests.insert(key, m); + } + + ParsedCatalog { + manifests, + bundles: raw.bundles.bundles, + } + }) +} + +/// Load all embedded extension manifests, keyed by `"tools/"` or `"channels/"`. +pub fn load_embedded() -> HashMap { + parsed_catalog().manifests.clone() +} + +/// Load embedded bundle definitions. +pub fn load_embedded_bundles() -> HashMap { + parsed_catalog().bundles.clone() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_load_embedded_parses() { + let manifests = load_embedded(); + // Should have at least the manifests from registry/ if built from the repo + // (empty is also valid for minimal builds without registry/) + assert!( + manifests.is_empty() || manifests.contains_key("tools/github"), + "Expected either empty catalog or github tool, got {} entries", + manifests.len() + ); + } + + #[test] + fn test_load_embedded_bundles_parses() { + let bundles = load_embedded_bundles(); + assert!( + bundles.is_empty() || bundles.contains_key("default"), + "Expected either empty bundles or 'default' bundle" + ); + } +} diff --git a/src/registry/installer.rs b/src/registry/installer.rs index 87de6330..0f7f84f0 100644 --- a/src/registry/installer.rs +++ b/src/registry/installer.rs @@ -137,6 +137,10 @@ impl RegistryInstaller { } /// Download and install a pre-built artifact. + /// + /// Supports two formats: + /// - **tar.gz bundle**: Contains `{name}.wasm` + `{name}.capabilities.json` + /// - **bare .wasm file**: Just the WASM binary (capabilities fetched separately if available) pub async fn install_from_artifact( &self, manifest: &ExtensionManifest, @@ -156,13 +160,6 @@ impl RegistryInstaller { )) })?; - let expected_sha = artifact.sha256.as_ref().ok_or_else(|| { - RegistryError::ExtensionNotFound(format!( - "No SHA256 hash for '{}'. Cannot verify download.", - manifest.name - )) - })?; - let target_dir = match manifest.kind { ManifestKind::Tool => &self.tools_dir, ManifestKind::Channel => &self.channels_dir, @@ -186,75 +183,90 @@ impl RegistryInstaller { "Downloading {} '{}'...", manifest.kind, manifest.display_name ); - let response = reqwest::get(url) - .await - .map_err(|e| RegistryError::DownloadFailed { - url: url.clone(), - reason: format!("request failed: {}", e), - })?; + let bytes = download_artifact(url).await?; - let response = response - .error_for_status() - .map_err(|e| RegistryError::DownloadFailed { - url: url.clone(), - reason: e.to_string(), - })?; - - let bytes = response - .bytes() - .await - .map_err(|e| RegistryError::DownloadFailed { - url: url.clone(), - reason: format!("failed to read body: {}", e), - })?; - - // Verify SHA256 - use sha2::{Digest, Sha256}; - let mut hasher = Sha256::new(); - hasher.update(&bytes); - let actual_sha = format!("{:x}", hasher.finalize()); - - if actual_sha != *expected_sha { - return Err(RegistryError::DownloadFailed { - url: url.clone(), - reason: format!( - "SHA256 mismatch: expected {}, got {}", - expected_sha, actual_sha - ), - }); + // Verify SHA256 if provided, warn otherwise + if let Some(expected_sha) = &artifact.sha256 { + verify_sha256(&bytes, expected_sha, url)?; + } else { + println!( + "WARNING: No SHA256 checksum for '{}'; download is not cryptographically verified.", + manifest.name + ); } - // Write file - fs::write(&target_wasm, &bytes) - .await - .map_err(RegistryError::Io)?; - - // Copy capabilities from source dir (still needed even for pre-built artifacts). - // NOTE: This requires the source tree to be present. When pre-built artifact - // distribution is implemented, capabilities should be bundled with the artifact - // or fetched from a separate URL. - let caps_source = self - .repo_root - .join(&manifest.source.dir) - .join(&manifest.source.capabilities); let target_caps = target_dir.join(format!("{}.capabilities.json", manifest.name)); - let has_capabilities = if caps_source.exists() { - fs::copy(&caps_source, &target_caps) + + // Detect format and extract + let has_capabilities = if is_gzip(&bytes) { + // tar.gz bundle: extract {name}.wasm and {name}.capabilities.json + let extracted = + extract_tar_gz(&bytes, &manifest.name, &target_wasm, &target_caps, url)?; + extracted.has_capabilities + } else { + // Bare WASM file + fs::write(&target_wasm, &bytes) .await .map_err(RegistryError::Io)?; - true - } else { - false + + // Try to get capabilities from: + // 1. Separate capabilities_url in the artifact + // 2. Source tree (legacy, requires repo) + if let Some(ref caps_url) = artifact.capabilities_url { + const MAX_CAPS_SIZE: usize = 1024 * 1024; // 1 MB + match download_artifact(caps_url).await { + Ok(caps_bytes) if caps_bytes.len() <= MAX_CAPS_SIZE => { + fs::write(&target_caps, &caps_bytes) + .await + .map_err(RegistryError::Io)?; + true + } + Ok(caps_bytes) => { + tracing::warn!( + "Capabilities file too large ({} bytes, max {}), skipping", + caps_bytes.len(), + MAX_CAPS_SIZE + ); + false + } + Err(e) => { + tracing::warn!("Failed to download capabilities from {}: {}", caps_url, e); + false + } + } + } else { + // Legacy fallback: try source tree + let caps_source = self + .repo_root + .join(&manifest.source.dir) + .join(&manifest.source.capabilities); + if caps_source.exists() { + fs::copy(&caps_source, &target_caps) + .await + .map_err(RegistryError::Io)?; + true + } else { + false + } + } }; println!(" Installed to {}", target_wasm.display()); + let mut warnings = Vec::new(); + if !has_capabilities { + warnings.push(format!( + "No capabilities file found for '{}'. Auth and hooks may not work.", + manifest.name + )); + } + Ok(InstallOutcome { name: manifest.name.clone(), kind: manifest.kind, wasm_path: target_wasm, has_capabilities, - warnings: Vec::new(), + warnings, }) } @@ -399,6 +411,159 @@ async fn build_wasm_component(source_dir: &Path, crate_name: &str) -> anyhow::Re ) } +/// Download an artifact from a URL. +async fn download_artifact(url: &str) -> Result { + let response = reqwest::get(url) + .await + .map_err(|e| RegistryError::DownloadFailed { + url: url.to_string(), + reason: format!("request failed: {}", e), + })?; + + let response = response + .error_for_status() + .map_err(|e| RegistryError::DownloadFailed { + url: url.to_string(), + reason: e.to_string(), + })?; + + response + .bytes() + .await + .map_err(|e| RegistryError::DownloadFailed { + url: url.to_string(), + reason: format!("failed to read body: {}", e), + }) +} + +/// Verify SHA256 of downloaded bytes. +fn verify_sha256(bytes: &[u8], expected: &str, url: &str) -> Result<(), RegistryError> { + use sha2::{Digest, Sha256}; + let mut hasher = Sha256::new(); + hasher.update(bytes); + let actual = format!("{:x}", hasher.finalize()); + + if actual != expected { + return Err(RegistryError::DownloadFailed { + url: url.to_string(), + reason: format!("SHA256 mismatch: expected {}, got {}", expected, actual), + }); + } + Ok(()) +} + +/// Check if bytes start with gzip magic number (0x1f 0x8b). +fn is_gzip(bytes: &[u8]) -> bool { + bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b +} + +/// Result of extracting a tar.gz bundle. +struct ExtractResult { + has_capabilities: bool, +} + +/// Extract a tar.gz archive, looking for `{name}.wasm` and `{name}.capabilities.json`. +fn extract_tar_gz( + bytes: &[u8], + name: &str, + target_wasm: &Path, + target_caps: &Path, + url: &str, +) -> Result { + use flate2::read::GzDecoder; + use tar::Archive; + + use std::io::Read as _; + + let decoder = GzDecoder::new(bytes); + let mut archive = Archive::new(decoder); + // Defense-in-depth: do not preserve permissions or extended attributes + archive.set_preserve_permissions(false); + #[cfg(any(unix, target_os = "redox"))] + archive.set_unpack_xattrs(false); + + // 100 MB cap on decompressed entry size to prevent decompression bombs + const MAX_ENTRY_SIZE: u64 = 100 * 1024 * 1024; + + let wasm_filename = format!("{}.wasm", name); + let caps_filename = format!("{}.capabilities.json", name); + let mut found_wasm = false; + let mut found_caps = false; + + let entries = archive + .entries() + .map_err(|e| RegistryError::DownloadFailed { + url: url.to_string(), + reason: format!("failed to read tar.gz entries: {}", e), + })?; + + for entry in entries { + let mut entry = entry.map_err(|e| RegistryError::DownloadFailed { + url: url.to_string(), + reason: format!("failed to read tar.gz entry: {}", e), + })?; + + if entry.size() > MAX_ENTRY_SIZE { + return Err(RegistryError::DownloadFailed { + url: url.to_string(), + reason: format!( + "archive entry too large ({} bytes, max {} bytes)", + entry.size(), + MAX_ENTRY_SIZE + ), + }); + } + + let entry_path = entry + .path() + .map_err(|e| RegistryError::DownloadFailed { + url: url.to_string(), + reason: format!("invalid path in tar.gz: {}", e), + })? + .to_path_buf(); + + // Match by filename (ignoring any directory prefix in the archive) + let filename = entry_path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or(""); + + if filename == wasm_filename { + let mut data = Vec::with_capacity(entry.size() as usize); + std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data) + .map_err(|e| RegistryError::DownloadFailed { + url: url.to_string(), + reason: format!("failed to read {} from archive: {}", wasm_filename, e), + })?; + std::fs::write(target_wasm, &data).map_err(RegistryError::Io)?; + found_wasm = true; + } else if filename == caps_filename { + let mut data = Vec::with_capacity(entry.size() as usize); + std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data) + .map_err(|e| RegistryError::DownloadFailed { + url: url.to_string(), + reason: format!("failed to read {} from archive: {}", caps_filename, e), + })?; + std::fs::write(target_caps, &data).map_err(RegistryError::Io)?; + found_caps = true; + } + } + + if !found_wasm { + return Err(RegistryError::DownloadFailed { + url: url.to_string(), + reason: format!( + "tar.gz archive does not contain '{}'. Archive may be malformed.", + wasm_filename + ), + }); + } + + Ok(ExtractResult { + has_capabilities: found_caps, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -412,4 +577,103 @@ mod tests { ); assert_eq!(installer.repo_root, PathBuf::from("/repo")); } + + #[test] + fn test_is_gzip() { + assert!(is_gzip(&[0x1f, 0x8b, 0x08])); + assert!(!is_gzip(&[0x00, 0x61, 0x73, 0x6d])); // WASM magic + assert!(!is_gzip(&[0x1f])); // Too short + assert!(!is_gzip(&[])); + } + + #[test] + fn test_verify_sha256_valid() { + use sha2::{Digest, Sha256}; + let data = b"hello world"; + let mut hasher = Sha256::new(); + hasher.update(data); + let hash = format!("{:x}", hasher.finalize()); + assert!(verify_sha256(data, &hash, "test://url").is_ok()); + } + + #[test] + fn test_verify_sha256_invalid() { + assert!(verify_sha256(b"data", "0000", "test://url").is_err()); + } + + #[test] + fn test_extract_tar_gz() { + use flate2::Compression; + use flate2::write::GzEncoder; + use tar::Builder; + + // Create a tar.gz in memory with test.wasm and test.capabilities.json + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + { + let mut builder = Builder::new(&mut encoder); + + let wasm_data = b"\0asm\x01\x00\x00\x00"; + let mut header = tar::Header::new_gnu(); + header.set_size(wasm_data.len() as u64); + header.set_cksum(); + builder + .append_data(&mut header, "test.wasm", &wasm_data[..]) + .unwrap(); + + let caps_data = br#"{"auth":null}"#; + let mut header = tar::Header::new_gnu(); + header.set_size(caps_data.len() as u64); + header.set_cksum(); + builder + .append_data(&mut header, "test.capabilities.json", &caps_data[..]) + .unwrap(); + + builder.finish().unwrap(); + } + let gz_bytes = encoder.finish().unwrap(); + + let tmp = tempfile::tempdir().unwrap(); + let wasm_path = tmp.path().join("test.wasm"); + let caps_path = tmp.path().join("test.capabilities.json"); + + let result = + extract_tar_gz(&gz_bytes, "test", &wasm_path, &caps_path, "test://url").unwrap(); + + assert!(wasm_path.exists()); + assert!(caps_path.exists()); + assert!(result.has_capabilities); + } + + #[test] + fn test_extract_tar_gz_missing_wasm() { + use flate2::Compression; + use flate2::write::GzEncoder; + use tar::Builder; + + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + { + let mut builder = Builder::new(&mut encoder); + + let data = b"not a wasm file"; + let mut header = tar::Header::new_gnu(); + header.set_size(data.len() as u64); + header.set_cksum(); + builder + .append_data(&mut header, "wrong.wasm", &data[..]) + .unwrap(); + builder.finish().unwrap(); + } + let gz_bytes = encoder.finish().unwrap(); + + let tmp = tempfile::tempdir().unwrap(); + let result = extract_tar_gz( + &gz_bytes, + "test", + &tmp.path().join("test.wasm"), + &tmp.path().join("test.capabilities.json"), + "test://url", + ); + + assert!(result.is_err()); + } } diff --git a/src/registry/manifest.rs b/src/registry/manifest.rs index 4a1c5591..2e93ed4e 100644 --- a/src/registry/manifest.rs +++ b/src/registry/manifest.rs @@ -88,10 +88,17 @@ pub struct SourceSpec { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ArtifactSpec { /// Download URL (null until release). + /// Can point to a `.wasm` file or a `.tar.gz` bundle containing both + /// `{name}.wasm` and `{name}.capabilities.json`. pub url: Option, - /// Hex SHA256 of the WASM binary (null until release). + /// Hex SHA256 of the downloaded artifact (null until release). pub sha256: Option, + + /// Optional separate download URL for the capabilities file. + /// Only needed when `url` points to a bare `.wasm` file instead of a bundle. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub capabilities_url: Option, } /// Summary of authentication requirements extracted from capabilities. @@ -138,7 +145,7 @@ pub struct BundleDefinition { } /// Top-level structure of `_bundles.json`. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct BundlesFile { pub bundles: std::collections::HashMap, } @@ -147,9 +154,24 @@ impl ExtensionManifest { /// Convert this manifest into a [`RegistryEntry`] for use with the in-chat /// extension discovery system. pub fn to_registry_entry(&self) -> RegistryEntry { - let source = ExtensionSource::WasmBuildable { - repo_url: self.source.dir.clone(), - build_dir: Some(self.source.dir.clone()), + // Prefer pre-built artifact download when a URL is available + let source = if let Some(artifact) = self.artifacts.get("wasm32-wasip2") { + if let Some(ref url) = artifact.url { + ExtensionSource::WasmDownload { + wasm_url: url.clone(), + capabilities_url: artifact.capabilities_url.clone(), + } + } else { + ExtensionSource::WasmBuildable { + repo_url: self.source.dir.clone(), + build_dir: Some(self.source.dir.clone()), + } + } + } else { + ExtensionSource::WasmBuildable { + repo_url: self.source.dir.clone(), + build_dir: Some(self.source.dir.clone()), + } }; let auth_hint = match self.auth_summary.as_ref().and_then(|a| a.method.as_deref()) { diff --git a/src/registry/mod.rs b/src/registry/mod.rs index a86fb5fc..b223c130 100644 --- a/src/registry/mod.rs +++ b/src/registry/mod.rs @@ -12,6 +12,7 @@ //! ``` pub mod catalog; +pub mod embedded; pub mod installer; pub mod manifest; diff --git a/src/secrets/mod.rs b/src/secrets/mod.rs index 8547fbfc..323f17c9 100644 --- a/src/secrets/mod.rs +++ b/src/secrets/mod.rs @@ -74,5 +74,4 @@ pub use types::{ SecretError, SecretRef, }; -#[cfg(test)] -pub use store::testing::InMemorySecretsStore; +pub use store::in_memory::InMemorySecretsStore; diff --git a/src/secrets/store.rs b/src/secrets/store.rs index d9eb6d79..4fbeddd4 100644 --- a/src/secrets/store.rs +++ b/src/secrets/store.rs @@ -635,9 +635,10 @@ fn libsql_row_to_secret(row: &libsql::Row) -> Result { }) } -/// In-memory implementation for testing. -#[cfg(test)] -pub mod testing { +/// In-memory secrets store. Used for testing and as a fallback when no +/// persistent secrets backend is configured (extension listing/install still +/// works, but stored secrets won't survive a restart). +pub mod in_memory { use std::collections::HashMap; use std::sync::Arc; @@ -794,7 +795,7 @@ mod tests { use crate::secrets::crypto::SecretsCrypto; use crate::secrets::store::SecretsStore; - use crate::secrets::store::testing::InMemorySecretsStore; + use crate::secrets::store::in_memory::InMemorySecretsStore; use crate::secrets::types::CreateSecretParams; fn test_store() -> InMemorySecretsStore { diff --git a/src/settings.rs b/src/settings.rs index b1b12347..3ca28b3f 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -1273,9 +1273,11 @@ mod tests { let from_db = Settings::from_db_map(&db_map); // Step 1 of the new wizard run: user enters a NEW database_url - let mut step1_settings = Settings::default(); - step1_settings.database_backend = Some("postgres".to_string()); - step1_settings.database_url = Some("postgres://new-host/ironclaw".to_string()); + let step1_settings = Settings { + database_backend: Some("postgres".to_string()), + database_url: Some("postgres://new-host/ironclaw".to_string()), + ..Settings::default() + }; // Wizard flow: load DB → merge_from(step1_overrides) let mut current = step1_settings.clone(); diff --git a/src/setup/wizard.rs b/src/setup/wizard.rs index b7258407..34f82915 100644 --- a/src/setup/wizard.rs +++ b/src/setup/wizard.rs @@ -2566,40 +2566,10 @@ fn build_channel_options(discovered: &[(String, ChannelCapabilitiesFile)]) -> Ve names } -/// Try to load the registry catalog. Returns None if the registry directory -/// cannot be found (e.g. running from an installed binary without the repo). +/// Try to load the registry catalog. Falls back to embedded manifests when +/// the `registry/` directory cannot be found (e.g. running from an installed binary). fn load_registry_catalog() -> Option { - // Try relative to current directory (dev usage) - let cwd = std::env::current_dir().ok()?; - let candidate = cwd.join("registry"); - if candidate.is_dir() { - return crate::registry::catalog::RegistryCatalog::load(&candidate).ok(); - } - - // Try relative to executable - if let Ok(exe) = std::env::current_exe() - && let Some(parent) = exe.parent() - { - let candidate = parent.join("registry"); - if candidate.is_dir() { - return crate::registry::catalog::RegistryCatalog::load(&candidate).ok(); - } - if let Some(grandparent) = parent.parent() { - let candidate = grandparent.join("registry"); - if candidate.is_dir() { - return crate::registry::catalog::RegistryCatalog::load(&candidate).ok(); - } - } - } - - // Try CARGO_MANIFEST_DIR (compile-time, works in dev builds) - let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")); - let candidate = manifest_dir.join("registry"); - if candidate.is_dir() { - return crate::registry::catalog::RegistryCatalog::load(&candidate).ok(); - } - - None + crate::registry::catalog::RegistryCatalog::load_or_embedded().ok() } /// Install selected channels from the registry that aren't already on disk diff --git a/src/tools/builtin/extension_tools.rs b/src/tools/builtin/extension_tools.rs index a22f6182..458fa927 100644 --- a/src/tools/builtin/extension_tools.rs +++ b/src/tools/builtin/extension_tools.rs @@ -596,6 +596,7 @@ mod tests { None, "test".to_string(), None, + Vec::new(), )) } } diff --git a/tests/openai_compat_integration.rs b/tests/openai_compat_integration.rs index 32623150..f8b8631a 100644 --- a/tests/openai_compat_integration.rs +++ b/tests/openai_compat_integration.rs @@ -198,6 +198,7 @@ async fn start_test_server_with_provider( skill_registry: None, skill_catalog: None, chat_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(30, 60), + registry_entries: Vec::new(), cost_guard: None, startup_time: std::time::Instant::now(), }); @@ -685,6 +686,7 @@ async fn test_no_llm_provider_returns_503() { skill_registry: None, skill_catalog: None, chat_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(30, 60), + registry_entries: Vec::new(), cost_guard: None, startup_time: std::time::Instant::now(), }); diff --git a/tests/ws_gateway_integration.rs b/tests/ws_gateway_integration.rs index cb002f2f..beb01859 100644 --- a/tests/ws_gateway_integration.rs +++ b/tests/ws_gateway_integration.rs @@ -56,6 +56,7 @@ async fn start_test_server() -> ( skill_registry: None, skill_catalog: None, chat_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(30, 60), + registry_entries: Vec::new(), cost_guard: None, startup_time: std::time::Instant::now(), });