gitops/optimclaw
1
0
Fork 0
mirror of https://github.com/outbackdingo/optimclaw.git synced 2026-08-25 14:53:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: embedded registry catalog and WASM bundle install pipeline (#283)

Browse Source 
* feat: embedded registry catalog and WASM bundle install pipeline

Embed registry manifests at compile time so the extension catalog is
available without network access. Add tar.gz bundle support for WASM
extension downloads (tools and channels), a /api/extensions/registry
endpoint, CI job to build and publish WASM bundles on release, and
ephemeral in-memory secrets fallback so the extension manager works
even without a persistent secrets store.

Key changes:
- build.rs: collect registry/*.json into embedded_catalog.json at compile time
- src/registry/embedded.rs + catalog.rs: load embedded or on-disk catalog
- src/extensions/manager.rs: download_and_install_wasm handles tar.gz bundles,
  bare .wasm files, and separate capabilities downloads; wasm channel install
- src/channels/web/server.rs: /api/extensions/registry endpoint + no-cache headers
- src/app.rs: ephemeral InMemorySecretsStore fallback for extension manager
- registry/*.json: populate artifact download URLs for release bundles
- .github/workflows/release.yml: build-wasm-extensions CI job
- Simplified setup wizard and CLI registry commands

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix: address PR review — archive hardening, decompression bomb guard, test fix

- Add 100 MB decompressed entry size cap to tar.gz extraction in both
  manager.rs and installer.rs to prevent decompression bombs
- Add archive.set_preserve_permissions(false) and set_unpack_xattrs(false)
  for defense-in-depth against malicious archives
- Fix test assertion logic in catalog.rs (|| → || with correct negation)
- Replace silent tar fallback in CI with explicit if/else for capabilities
- Add warning when installing without SHA256 verification

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix: resolve clippy warning in settings.rs and enforce zero-warnings policy

Use struct initializer with ..Default::default() instead of field
reassignment. Update CLAUDE.md to codify zero clippy warnings policy —
all warnings must be fixed before committing, including pre-existing ones.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix: address PR review round 2 — build reliability, caps validation, naming

- build.rs: emit per-file rerun-if-changed for reliable content tracking;
  fix bundles fallback to match BundlesFile shape ({"bundles":{}})
- embedded.rs: parse catalog once via OnceLock instead of double-parsing
- manager.rs + installer.rs: add 1 MB size cap on capabilities_url downloads
  with proper error surfacing
- secrets/store.rs: rename misleading `pub mod testing` to `pub mod in_memory`
- server.rs: track installed extensions by (name, kind) tuple to avoid
  false positives across different extension kinds

Co-Authored-By: Claude Opus 4.6 <[email protected]>

---------

Co-authored-by: Claude Opus 4.6 <[email protected]>
This commit is contained in:
Illia Polosukhin
2026-02-21 05:43:28 +00:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 3d4c647216
commit 436066415b
40 changed files with 1406 additions and 257 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/app.rs
+37 -9
View File
@@ -579,11 +579,44 @@ impl AppBuilder {
tokio::join!(wasm_tools_future, mcp_servers_future);
// Create extension manager
let extension_manager = if let Some(ref secrets) = self.secrets_store {
// Load registry catalog entries for extension discovery
let catalog_entries = match crate::registry::RegistryCatalog::load_or_embedded() {
Ok(catalog) => {
let entries: Vec<_> = catalog
.all()
.iter()
.map(|m| m.to_registry_entry())
.collect();
tracing::info!(
count = entries.len(),
"Loaded registry catalog entries for extension discovery"
);
entries
}
Err(e) => {
tracing::warn!("Failed to load registry catalog: {}", e);
Vec::new()
}
};
// Create extension manager. Use ephemeral in-memory secrets if no
// persistent store is configured (listing/install/activate still work).
let ext_secrets: Arc<dyn crate::secrets::SecretsStore + Send + Sync> = if let Some(ref s) =
self.secrets_store
{
Arc::clone(s)
} else {
use crate::secrets::{InMemorySecretsStore, SecretsCrypto};
let ephemeral_key =
secrecy::SecretString::from(crate::secrets::keychain::generate_master_key_hex());
let crypto = Arc::new(SecretsCrypto::new(ephemeral_key).expect("ephemeral crypto"));
tracing::debug!("Using ephemeral in-memory secrets store for extension manager");
Arc::new(InMemorySecretsStore::new(crypto))
};
let extension_manager = {
let manager = Arc::new(ExtensionManager::new(
Arc::clone(&mcp_session_manager),
Arc::clone(secrets),
ext_secrets,
Arc::clone(tools),
Some(Arc::clone(hooks)),
wasm_tool_runtime.clone(),
@@ -592,16 +625,11 @@ impl AppBuilder {
self.config.tunnel.public_url.clone(),
"default".to_string(),
self.db.clone(),
catalog_entries.clone(),
));
tools.register_extension_tools(Arc::clone(&manager));
tracing::info!("Extension manager initialized with in-chat discovery tools");
Some(manager)
} else {
tracing::debug!(
"Extension manager not available (no secrets store). \
Extension tools won't be registered."
);
None
};
// register_builder_tool() already calls register_dev_tools() internally,
src/channels/web/mod.rs
+8
View File
@@ -89,6 +89,7 @@ impl GatewayChannel {
skill_registry: None,
skill_catalog: None,
chat_rate_limiter: server::RateLimiter::new(30, 60),
registry_entries: Vec::new(),
cost_guard: None,
startup_time: std::time::Instant::now(),
});
@@ -121,6 +122,7 @@ impl GatewayChannel {
skill_registry: self.state.skill_registry.clone(),
skill_catalog: self.state.skill_catalog.clone(),
chat_rate_limiter: server::RateLimiter::new(30, 60),
registry_entries: self.state.registry_entries.clone(),
cost_guard: self.state.cost_guard.clone(),
startup_time: self.state.startup_time,
};
@@ -210,6 +212,12 @@ impl GatewayChannel {
self
}
/// Inject registry catalog entries for the available extensions API.
pub fn with_registry_entries(mut self, entries: Vec<crate::extensions::RegistryEntry>) -> Self {
self.rebuild_state(|s| s.registry_entries = entries);
self
}
/// Inject the cost guard for token/cost tracking in the status popover.
pub fn with_cost_guard(mut self, cg: Arc<crate::agent::cost_guard::CostGuard>) -> Self {
self.rebuild_state(|s| s.cost_guard = Some(cg));
src/channels/web/server.rs
+107 -9
View File
@@ -13,7 +13,7 @@ use axum::{
http::{StatusCode, header},
middleware,
response::{
Html, IntoResponse,
IntoResponse,
sse::{Event, KeepAlive, Sse},
},
routing::{get, post},
@@ -148,6 +148,9 @@ pub struct GatewayState {
pub skill_catalog: Option<Arc<crate::skills::catalog::SkillCatalog>>,
/// Rate limiter for chat endpoints (30 messages per 60 seconds).
pub chat_rate_limiter: RateLimiter,
/// Registry catalog entries for the available extensions API.
/// Populated at startup from `registry/` manifests, independent of extension manager.
pub registry_entries: Vec<crate::extensions::RegistryEntry>,
/// Cost guard for token/cost tracking.
pub cost_guard: Option<Arc<crate::agent::cost_guard::CostGuard>>,
/// Server startup time for uptime calculation.
@@ -218,6 +221,7 @@ pub async fn start_server(
// Extensions
.route("/api/extensions", get(extensions_list_handler))
.route("/api/extensions/tools", get(extensions_tools_handler))
.route("/api/extensions/registry", get(extensions_registry_handler))
.route("/api/extensions/install", post(extensions_install_handler))
.route(
"/api/extensions/{name}/activate",
@@ -348,20 +352,32 @@ pub async fn start_server(
// --- Static file handlers ---
async fn index_handler() -> Html<&'static str> {
Html(include_str!("static/index.html"))
async fn index_handler() -> impl IntoResponse {
(
[
(header::CONTENT_TYPE, "text/html; charset=utf-8"),
(header::CACHE_CONTROL, "no-cache"),
],
include_str!("static/index.html"),
)
}
async fn css_handler() -> impl IntoResponse {
(
[(header::CONTENT_TYPE, "text/css")],
[
(header::CONTENT_TYPE, "text/css"),
(header::CACHE_CONTROL, "no-cache"),
],
include_str!("static/style.css"),
)
}
async fn js_handler() -> impl IntoResponse {
(
[(header::CONTENT_TYPE, "application/javascript")],
[
(header::CONTENT_TYPE, "application/javascript"),
(header::CACHE_CONTROL, "no-cache"),
],
include_str!("static/app.js"),
)
}
@@ -1722,10 +1738,30 @@ async fn extensions_install_handler(
State(state): State<Arc<GatewayState>>,
Json(req): Json<InstallExtensionRequest>,
) -> Result<Json<ActionResponse>, (StatusCode, String)> {
let ext_mgr = state.extension_manager.as_ref().ok_or((
StatusCode::NOT_IMPLEMENTED,
"Extension manager not available (secrets store required)".to_string(),
))?;
// When extension manager isn't available, check registry entries for a helpful message
let Some(ext_mgr) = state.extension_manager.as_ref() else {
// Look up the entry in the catalog to give a specific error
if let Some(entry) = state.registry_entries.iter().find(|e| e.name == req.name) {
let msg = match &entry.source {
crate::extensions::ExtensionSource::WasmBuildable { .. } => {
format!(
"'{}' requires building from source. \
Run `ironclaw registry install {}` from the CLI.",
req.name, req.name
)
}
_ => format!(
"Extension manager not available (secrets store required). \
Configure DATABASE_URL or a secrets backend to enable installation of '{}'.",
req.name
),
};
return Ok(Json(ActionResponse::fail(msg)));
}
return Ok(Json(ActionResponse::fail(
"Extension manager not available (secrets store required)".to_string(),
)));
};
let kind_hint = req.kind.as_deref().and_then(|k| match k {
"mcp_server" => Some(crate::extensions::ExtensionKind::McpServer),
@@ -1874,6 +1910,68 @@ async fn extensions_remove_handler(
}
}
async fn extensions_registry_handler(
State(state): State<Arc<GatewayState>>,
Query(params): Query<RegistrySearchQuery>,
) -> Json<RegistrySearchResponse> {
let query = params.query.unwrap_or_default();
let query_lower = query.to_lowercase();
let tokens: Vec<&str> = query_lower.split_whitespace().collect();
// Filter registry entries by query (or return all if empty)
let matching: Vec<&crate::extensions::RegistryEntry> = if tokens.is_empty() {
state.registry_entries.iter().collect()
} else {
state
.registry_entries
.iter()
.filter(|e| {
let name = e.name.to_lowercase();
let display = e.display_name.to_lowercase();
let desc = e.description.to_lowercase();
tokens.iter().any(|t| {
name.contains(t)
|| display.contains(t)
|| desc.contains(t)
|| e.keywords.iter().any(|k| k.to_lowercase().contains(t))
})
})
.collect()
};
// Cross-reference with installed extensions by (name, kind) to avoid
// false positives when the same name exists as different kinds.
let installed: std::collections::HashSet<(String, String)> =
if let Some(ext_mgr) = state.extension_manager.as_ref() {
ext_mgr
.list(None)
.await
.unwrap_or_default()
.into_iter()
.map(|ext| (ext.name, ext.kind.to_string()))
.collect()
} else {
std::collections::HashSet::new()
};
let entries = matching
.into_iter()
.map(|e| {
let kind_str = e.kind.to_string();
RegistryEntryInfo {
name: e.name.clone(),
display_name: e.display_name.clone(),
installed: installed.contains(&(e.name.clone(), kind_str.clone())),
kind: kind_str,
description: e.description.clone(),
keywords: e.keywords.clone(),
}
})
.collect();
Json(RegistrySearchResponse { entries })
}
// --- Skills handlers ---
async fn skills_list_handler(
src/channels/web/types.rs
+22
View File
@@ -408,6 +408,28 @@ impl ActionResponse {
}
}
// --- Registry ---
#[derive(Debug, Serialize)]
pub struct RegistryEntryInfo {
pub name: String,
pub display_name: String,
pub kind: String,
pub description: String,
pub keywords: Vec<String>,
pub installed: bool,
}
#[derive(Debug, Serialize)]
pub struct RegistrySearchResponse {
pub entries: Vec<RegistryEntryInfo>,
}
#[derive(Debug, Deserialize)]
pub struct RegistrySearchQuery {
pub query: Option<String>,
}
// --- Skills ---
#[derive(Debug, Serialize)]
src/channels/web/ws.rs
+1
View File
@@ -490,6 +490,7 @@ mod tests {
skill_registry: None,
skill_catalog: None,
chat_rate_limiter: crate::channels::web::server::RateLimiter::new(30, 60),
registry_entries: Vec::new(),
cost_guard: None,
startup_time: std::time::Instant::now(),
}
src/cli/registry.rs
+17 -51
View File
@@ -1,7 +1,5 @@
//! Registry CLI commands for discovering and installing extensions.
use std::path::PathBuf;
use clap::Subcommand;
use crate::registry::catalog::RegistryCatalog;
@@ -59,8 +57,20 @@ pub enum RegistryCommand {
/// Run a registry command.
pub async fn run_registry_command(cmd: RegistryCommand) -> anyhow::Result<()> {
let registry_dir = find_registry_dir()?;
let catalog = RegistryCatalog::load(&registry_dir)?;
// For install commands that need to build from source, a disk registry is required.
// For list/info, embedded manifests suffice.
let registry_dir = RegistryCatalog::find_dir();
let catalog = if let Some(ref dir) = registry_dir {
RegistryCatalog::load(dir)?
} else {
RegistryCatalog::load_or_embedded()?
};
// Resolve repo root for installer (empty path when running from binary)
let repo_root = registry_dir
.as_ref()
.and_then(|d| d.parent().map(|p| p.to_path_buf()))
.unwrap_or_default();
match cmd {
RegistryCommand::List { kind, tag, verbose } => {
@@ -68,53 +78,14 @@ pub async fn run_registry_command(cmd: RegistryCommand) -> anyhow::Result<()> {
}
RegistryCommand::Info { name } => cmd_info(&catalog, &name),
RegistryCommand::Install { name, force, build } => {
cmd_install(&catalog, &registry_dir, &name, force, build).await
cmd_install(&catalog, &repo_root, &name, force, build).await
}
RegistryCommand::InstallDefaults { force, build } => {
cmd_install(&catalog, &registry_dir, "default", force, build).await
cmd_install(&catalog, &repo_root, "default", force, build).await
}
}
}
/// Find the registry directory by looking relative to the current executable or cwd.
fn find_registry_dir() -> anyhow::Result<PathBuf> {
// Try relative to current directory (for dev usage)
let cwd = std::env::current_dir()?;
let candidate = cwd.join("registry");
if candidate.is_dir() {
return Ok(candidate);
}
// Try relative to executable (covers installed binary, target/debug/, target/release/)
if let Ok(exe) = std::env::current_exe()
&& let Some(parent) = exe.parent()
{
// Walk up to 3 levels: exe dir, parent (target/release → target), grandparent (→ repo root)
let mut dir = Some(parent);
for _ in 0..3 {
if let Some(d) = dir {
let candidate = d.join("registry");
if candidate.is_dir() {
return Ok(candidate);
}
dir = d.parent();
}
}
}
// Try CARGO_MANIFEST_DIR (compile-time, works in dev builds)
let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
let candidate = manifest_dir.join("registry");
if candidate.is_dir() {
return Ok(candidate);
}
anyhow::bail!(
"Could not find registry/ directory. Run from the ironclaw repo root, \
or ensure registry/ is next to the ironclaw binary."
)
}
fn cmd_list(
catalog: &RegistryCatalog,
kind: Option<&str>,
@@ -254,16 +225,11 @@ fn cmd_info(catalog: &RegistryCatalog, name: &str) -> anyhow::Result<()> {
async fn cmd_install(
catalog: &RegistryCatalog,
registry_dir: &std::path::Path,
repo_root: &std::path::Path,
name: &str,
force: bool,
prefer_build: bool,
) -> anyhow::Result<()> {
// Registry dir parent is the repo root
let repo_root = registry_dir
.parent()
.ok_or_else(|| anyhow::anyhow!("Cannot determine repo root from registry dir"))?;
let installer = RegistryInstaller::with_defaults(repo_root.to_path_buf());
let (manifests, bundle) = catalog.resolve(name)?;
src/extensions/manager.rs
+283 -53
View File
@@ -75,9 +75,15 @@ impl ExtensionManager {
tunnel_url: Option<String>,
user_id: String,
store: Option<Arc<dyn crate::db::Database>>,
catalog_entries: Vec<RegistryEntry>,
) -> Self {
let registry = if catalog_entries.is_empty() {
ExtensionRegistry::new()
} else {
ExtensionRegistry::new_with_catalog(catalog_entries)
};
Self {
registry: ExtensionRegistry::new(),
registry,
discovery: OnlineDiscovery::new(),
mcp_session_manager,
mcp_clients: RwLock::new(HashMap::new()),
@@ -131,9 +137,14 @@ impl ExtensionManager {
url: Option<&str>,
kind_hint: Option<ExtensionKind>,
) -> Result<InstallResult, ExtensionError> {
tracing::info!(extension = %name, url = ?url, kind = ?kind_hint, "Installing extension");
// If we have a registry entry, use it
if let Some(entry) = self.registry.get(name).await {
return self.install_from_entry(&entry).await;
return self.install_from_entry(&entry).await.map_err(|e| {
tracing::error!(extension = %name, error = %e, "Extension install failed");
e
});
}
// If a URL was provided, determine kind and install
@@ -143,19 +154,21 @@ impl ExtensionManager {
ExtensionKind::McpServer => self.install_mcp_from_url(name, url).await,
ExtensionKind::WasmTool => self.install_wasm_tool_from_url(name, url).await,
ExtensionKind::WasmChannel => {
Err(ExtensionError::InstallFailed(
"WASM channel installation from URL not yet supported. \
Place the .wasm and .capabilities.json files in ~/.ironclaw/channels/ and restart."
.to_string(),
))
self.install_wasm_channel_from_url(name, url, None).await
}
};
}
.map_err(|e| {
tracing::error!(extension = %name, url = %url, error = %e, "Extension install from URL failed");
e
});
}
Err(ExtensionError::NotFound(format!(
let err = ExtensionError::NotFound(format!(
"'{}' not found in registry. Try searching with discover:true or provide a URL.",
name
)))
));
tracing::warn!(extension = %name, "Extension not found in registry");
Err(err)
}
/// Authenticate an installed extension.
@@ -433,16 +446,51 @@ impl ExtensionManager {
self.install_mcp_from_url(&entry.name, &url).await
}
ExtensionKind::WasmTool => match &entry.source {
ExtensionSource::WasmDownload { wasm_url, .. } => {
self.install_wasm_tool_from_url(&entry.name, wasm_url).await
ExtensionSource::WasmDownload {
wasm_url,
capabilities_url,
} => {
self.install_wasm_tool_from_url_with_caps(
&entry.name,
wasm_url,
capabilities_url.as_deref(),
)
.await
}
ExtensionSource::WasmBuildable { .. } => {
Err(ExtensionError::InstallFailed(format!(
"'{}' requires building from source. Run `ironclaw registry install {}` \
from the CLI (requires cargo-component).",
entry.name, entry.name
)))
}
_ => Err(ExtensionError::InstallFailed(
"WASM tool entry has no download URL".to_string(),
)),
},
ExtensionKind::WasmChannel => Err(ExtensionError::InstallFailed(
"WASM channel installation not yet supported via this flow".to_string(),
)),
ExtensionKind::WasmChannel => match &entry.source {
ExtensionSource::WasmDownload {
wasm_url,
capabilities_url,
} => {
self.install_wasm_channel_from_url(
&entry.name,
wasm_url,
capabilities_url.as_deref(),
)
.await
}
ExtensionSource::WasmBuildable { .. } => {
Err(ExtensionError::InstallFailed(format!(
"'{}' requires building from source. Run `ironclaw registry install {}` \
from the CLI (requires cargo-component).",
entry.name, entry.name
)))
}
_ => Err(ExtensionError::InstallFailed(
"WASM channel entry has no download URL".to_string(),
)),
},
}
}
@@ -482,6 +530,57 @@ impl ExtensionManager {
name: &str,
url: &str,
) -> Result<InstallResult, ExtensionError> {
self.install_wasm_tool_from_url_with_caps(name, url, None)
.await
}
async fn install_wasm_tool_from_url_with_caps(
&self,
name: &str,
url: &str,
capabilities_url: Option<&str>,
) -> Result<InstallResult, ExtensionError> {
self.download_and_install_wasm(name, url, capabilities_url, &self.wasm_tools_dir)
.await?;
Ok(InstallResult {
name: name.to_string(),
kind: ExtensionKind::WasmTool,
message: format!("WASM tool '{}' installed. Run activate to load it.", name),
})
}
async fn install_wasm_channel_from_url(
&self,
name: &str,
url: &str,
capabilities_url: Option<&str>,
) -> Result<InstallResult, ExtensionError> {
self.download_and_install_wasm(name, url, capabilities_url, &self.wasm_channels_dir)
.await?;
Ok(InstallResult {
name: name.to_string(),
kind: ExtensionKind::WasmChannel,
message: format!(
"WASM channel '{}' installed to {}. Restart to activate.",
name,
self.wasm_channels_dir.display()
),
})
}
/// Download a WASM extension (tool or channel) from URL and install to target directory.
///
/// Handles both tar.gz bundles (containing `.wasm` + `.capabilities.json`) and bare
/// `.wasm` files. Validates HTTPS, size limits, and file format.
async fn download_and_install_wasm(
&self,
name: &str,
url: &str,
capabilities_url: Option<&str>,
target_dir: &std::path::Path,
) -> Result<(), ExtensionError> {
// Require HTTPS to prevent downgrade attacks
if !url.starts_with("https://") {
return Err(ExtensionError::InstallFailed(
@@ -490,33 +589,41 @@ impl ExtensionManager {
}
// 50 MB cap to prevent disk-fill DoS
const MAX_WASM_SIZE: usize = 50 * 1024 * 1024;
const MAX_DOWNLOAD_SIZE: usize = 50 * 1024 * 1024;
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(60))
.build()
.map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?;
let response = client
.get(url)
.send()
.await
.map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?;
tracing::debug!(extension = %name, url = %url, "Downloading WASM extension");
let response = client.get(url).send().await.map_err(|e| {
tracing::error!(extension = %name, url = %url, error = %e, "Download request failed");
ExtensionError::DownloadFailed(e.to_string())
})?;
if !response.status().is_success() {
let status = response.status();
tracing::error!(
extension = %name,
url = %url,
status = %status,
"Download returned non-success HTTP status"
);
return Err(ExtensionError::DownloadFailed(format!(
"HTTP {}",
response.status()
"HTTP {} from {}",
status, url
)));
}
// Check Content-Length header before downloading the full body
if let Some(len) = response.content_length()
&& len as usize > MAX_WASM_SIZE
&& len as usize > MAX_DOWNLOAD_SIZE
{
return Err(ExtensionError::InstallFailed(format!(
"WASM binary too large ({} bytes, max {} bytes)",
len, MAX_WASM_SIZE
"Download too large ({} bytes, max {} bytes)",
len, MAX_DOWNLOAD_SIZE
)));
}
@@ -525,45 +632,164 @@ impl ExtensionManager {
.await
.map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?;
if bytes.len() > MAX_WASM_SIZE {
if bytes.len() > MAX_DOWNLOAD_SIZE {
return Err(ExtensionError::InstallFailed(format!(
"WASM binary too large ({} bytes, max {} bytes)",
"Download too large ({} bytes, max {} bytes)",
bytes.len(),
MAX_WASM_SIZE
MAX_DOWNLOAD_SIZE
)));
}
// Basic WASM magic number check (\0asm)
if bytes.len() < 4 || &bytes[..4] != b"\0asm" {
return Err(ExtensionError::InstallFailed(
"Downloaded file is not a valid WASM binary (bad magic number)".to_string(),
));
// Ensure target directory exists
tokio::fs::create_dir_all(target_dir)
.await
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
let wasm_path = target_dir.join(format!("{}.wasm", name));
let caps_path = target_dir.join(format!("{}.capabilities.json", name));
// Detect format: gzip (tar.gz bundle) or bare WASM
if bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b {
// tar.gz bundle: extract {name}.wasm and {name}.capabilities.json
self.extract_wasm_tar_gz(name, &bytes, &wasm_path, &caps_path)?;
} else {
// Bare WASM file: validate magic number
if bytes.len() < 4 || &bytes[..4] != b"\0asm" {
return Err(ExtensionError::InstallFailed(
"Downloaded file is not a valid WASM binary (bad magic number)".to_string(),
));
}
tokio::fs::write(&wasm_path, &bytes)
.await
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
// Download capabilities separately if URL provided
if let Some(caps_url) = capabilities_url {
const MAX_CAPS_SIZE: usize = 1024 * 1024; // 1 MB
match client.get(caps_url).send().await {
Ok(resp) if resp.status().is_success() => match resp.bytes().await {
Ok(caps_bytes) if caps_bytes.len() <= MAX_CAPS_SIZE => {
if let Err(e) = tokio::fs::write(&caps_path, &caps_bytes).await {
tracing::warn!(
"Failed to write capabilities for '{}': {}",
name,
e
);
}
}
Ok(caps_bytes) => {
tracing::warn!(
"Capabilities file for '{}' too large ({} bytes, max {})",
name,
caps_bytes.len(),
MAX_CAPS_SIZE
);
}
Err(e) => {
tracing::warn!("Failed to download capabilities for '{}': {}", name, e);
}
},
_ => {
tracing::warn!(
"Failed to download capabilities for '{}' from {}",
name,
caps_url
);
}
}
}
}
// Ensure tools directory exists
tokio::fs::create_dir_all(&self.wasm_tools_dir)
.await
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
// Write the WASM file
let wasm_path = self.wasm_tools_dir.join(format!("{}.wasm", name));
tokio::fs::write(&wasm_path, &bytes)
.await
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
tracing::info!(
"Installed WASM tool '{}' ({} bytes) from {} to {}",
"Installed WASM extension '{}' from {} to {}",
name,
bytes.len(),
url,
wasm_path.display()
);
Ok(InstallResult {
name: name.to_string(),
kind: ExtensionKind::WasmTool,
message: format!("WASM tool '{}' installed. Run activate to load it.", name),
})
Ok(())
}
/// Extract a tar.gz bundle into the WASM tools directory.
fn extract_wasm_tar_gz(
&self,
name: &str,
bytes: &[u8],
target_wasm: &std::path::Path,
target_caps: &std::path::Path,
) -> Result<(), ExtensionError> {
use flate2::read::GzDecoder;
use tar::Archive;
use std::io::Read as _;
let decoder = GzDecoder::new(bytes);
let mut archive = Archive::new(decoder);
// Defense-in-depth: do not preserve permissions or extended attributes
archive.set_preserve_permissions(false);
#[cfg(any(unix, target_os = "redox"))]
archive.set_unpack_xattrs(false);
// 100 MB cap on decompressed entry size to prevent decompression bombs
const MAX_ENTRY_SIZE: u64 = 100 * 1024 * 1024;
let wasm_filename = format!("{}.wasm", name);
let caps_filename = format!("{}.capabilities.json", name);
let mut found_wasm = false;
let entries = archive
.entries()
.map_err(|e| ExtensionError::InstallFailed(format!("Bad tar.gz archive: {}", e)))?;
for entry in entries {
let mut entry = entry
.map_err(|e| ExtensionError::InstallFailed(format!("Bad tar.gz entry: {}", e)))?;
if entry.size() > MAX_ENTRY_SIZE {
return Err(ExtensionError::InstallFailed(format!(
"Archive entry too large ({} bytes, max {} bytes)",
entry.size(),
MAX_ENTRY_SIZE
)));
}
let entry_path = entry
.path()
.map_err(|e| {
ExtensionError::InstallFailed(format!("Invalid path in tar.gz: {}", e))
})?
.to_path_buf();
let filename = entry_path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
if filename == wasm_filename {
let mut data = Vec::with_capacity(entry.size() as usize);
std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data)
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
std::fs::write(target_wasm, &data)
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
found_wasm = true;
} else if filename == caps_filename {
let mut data = Vec::with_capacity(entry.size() as usize);
std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data)
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
std::fs::write(target_caps, &data)
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
}
}
if !found_wasm {
return Err(ExtensionError::InstallFailed(format!(
"tar.gz archive does not contain '{}'",
wasm_filename
)));
}
Ok(())
}
async fn auth_mcp(
@@ -1074,7 +1300,7 @@ impl ExtensionManager {
/// Infer the extension kind from a URL.
fn infer_kind_from_url(url: &str) -> ExtensionKind {
if url.ends_with(".wasm") {
if url.ends_with(".wasm") || url.ends_with(".tar.gz") {
ExtensionKind::WasmTool
} else {
ExtensionKind::McpServer
@@ -1092,6 +1318,10 @@ mod tests {
infer_kind_from_url("https://example.com/tool.wasm"),
ExtensionKind::WasmTool
);
assert_eq!(
infer_kind_from_url("https://example.com/tool-wasm32-wasip2.tar.gz"),
ExtensionKind::WasmTool
);
assert_eq!(
infer_kind_from_url("https://mcp.notion.com"),
ExtensionKind::McpServer
src/extensions/registry.rs
+92
View File
@@ -26,6 +26,26 @@ impl ExtensionRegistry {
}
}
/// Create a new registry merging builtin entries with catalog-provided entries.
///
/// Deduplicates by `(name, kind)` pair -- a builtin MCP "slack" and a registry
/// WASM "slack" can coexist since they're different kinds.
pub fn new_with_catalog(catalog_entries: Vec<RegistryEntry>) -> Self {
let mut entries = builtin_entries();
for entry in catalog_entries {
if !entries
.iter()
.any(|e| e.name == entry.name && e.kind == entry.kind)
{
entries.push(entry);
}
}
Self {
entries,
discovery_cache: RwLock::new(Vec::new()),
}
}
/// Search the registry by query string. Returns results sorted by relevance.
///
/// Splits the query into lowercase tokens and scores each entry by matches
@@ -542,4 +562,76 @@ mod tests {
let results = registry.search("dup").await;
assert_eq!(results.len(), 1, "Should not duplicate cached entries");
}
#[tokio::test]
async fn test_new_with_catalog() {
let catalog_entries = vec![
RegistryEntry {
name: "telegram".to_string(),
display_name: "Telegram".to_string(),
kind: ExtensionKind::WasmChannel,
description: "Telegram Bot API channel".to_string(),
keywords: vec!["messaging".into(), "bot".into()],
source: ExtensionSource::WasmBuildable {
repo_url: "channels-src/telegram".to_string(),
build_dir: Some("channels-src/telegram".to_string()),
},
auth_hint: AuthHint::CapabilitiesAuth,
},
// This shares a name with a builtin but has a different kind, so both should appear
RegistryEntry {
name: "slack".to_string(),
display_name: "Slack WASM".to_string(),
kind: ExtensionKind::WasmTool,
description: "Slack WASM tool".to_string(),
keywords: vec!["messaging".into()],
source: ExtensionSource::WasmBuildable {
repo_url: "tools-src/slack".to_string(),
build_dir: Some("tools-src/slack".to_string()),
},
auth_hint: AuthHint::CapabilitiesAuth,
},
];
let registry = ExtensionRegistry::new_with_catalog(catalog_entries);
// Should find the new telegram entry
let results = registry.search("telegram").await;
assert!(!results.is_empty(), "Should find telegram from catalog");
assert_eq!(results[0].entry.name, "telegram");
// Should have both builtin MCP slack and catalog WASM slack
let results = registry.search("slack").await;
let slack_mcp = results
.iter()
.any(|r| r.entry.name == "slack" && r.entry.kind == ExtensionKind::McpServer);
let slack_wasm = results
.iter()
.any(|r| r.entry.name == "slack" && r.entry.kind == ExtensionKind::WasmTool);
assert!(slack_mcp, "Should have builtin MCP slack");
assert!(slack_wasm, "Should have catalog WASM slack");
}
#[tokio::test]
async fn test_new_with_catalog_dedup_same_kind() {
// A catalog entry with same name AND kind as a builtin should be skipped
let catalog_entries = vec![RegistryEntry {
name: "slack".to_string(),
display_name: "Slack Override".to_string(),
kind: ExtensionKind::McpServer, // same kind as builtin
description: "Should be skipped".to_string(),
keywords: vec![],
source: ExtensionSource::McpUrl {
url: "https://other.slack.com".to_string(),
},
auth_hint: AuthHint::Dcr,
}];
let registry = ExtensionRegistry::new_with_catalog(catalog_entries);
let entry = registry.get("slack").await;
assert!(entry.is_some());
// Should still be the builtin, not the override
assert_eq!(entry.unwrap().display_name, "Slack");
}
}
src/main.rs
+39 -9
View File
@@ -909,11 +909,43 @@ async fn main() -> anyhow::Result<()> {
let (dev_loaded_tool_names, _) = tokio::join!(wasm_tools_future, mcp_servers_future);
// Create extension manager for in-chat discovery/install/auth/activate
let extension_manager = if let Some(ref secrets) = secrets_store {
// Load registry catalog entries for in-chat extension discovery
let catalog_entries = match ironclaw::registry::RegistryCatalog::load_or_embedded() {
Ok(catalog) => {
let entries: Vec<ironclaw::extensions::RegistryEntry> = catalog
.all()
.iter()
.map(|m| m.to_registry_entry())
.collect();
tracing::info!(
count = entries.len(),
"Loaded registry catalog entries for extension discovery"
);
entries
}
Err(e) => {
tracing::warn!("Failed to load registry catalog: {}", e);
Vec::new()
}
};
// Create extension manager for in-chat discovery/install/auth/activate.
// If no persistent secrets store is available, use an ephemeral in-memory store
// so that listing/installing/activating extensions still works (auth won't persist).
let ext_secrets: Arc<dyn SecretsStore + Send + Sync> = if let Some(ref s) = secrets_store {
Arc::clone(s)
} else {
use ironclaw::secrets::{InMemorySecretsStore, SecretsCrypto};
let ephemeral_key =
secrecy::SecretString::from(ironclaw::secrets::keychain::generate_master_key_hex());
let crypto = Arc::new(SecretsCrypto::new(ephemeral_key).expect("ephemeral crypto"));
tracing::debug!("Using ephemeral in-memory secrets store for extension manager");
Arc::new(InMemorySecretsStore::new(crypto))
};
let extension_manager = {
let manager = Arc::new(ExtensionManager::new(
Arc::clone(&mcp_session_manager),
Arc::clone(secrets),
ext_secrets,
Arc::clone(&tools),
Some(Arc::clone(&hooks)),
wasm_tool_runtime.clone(),
@@ -922,16 +954,11 @@ async fn main() -> anyhow::Result<()> {
config.tunnel.public_url.clone(),
"default".to_string(),
db.clone(),
catalog_entries.clone(),
));
tools.register_extension_tools(Arc::clone(&manager));
tracing::info!("Extension manager initialized with in-chat discovery tools");
Some(manager)
} else {
tracing::debug!(
"Extension manager not available (no secrets store). \
Extension tools won't be registered."
);
None
};
// Set up orchestrator for sandboxed job execution
@@ -1341,6 +1368,9 @@ async fn main() -> anyhow::Result<()> {
if let Some(ref ext_mgr) = extension_manager {
gw = gw.with_extension_manager(Arc::clone(ext_mgr));
}
if !catalog_entries.is_empty() {
gw = gw.with_registry_entries(catalog_entries.clone());
}
if let Some(ref d) = db {
gw = gw.with_store(Arc::clone(d));
}
src/registry/catalog.rs
+72
View File
@@ -3,6 +3,7 @@
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use crate::registry::embedded;
use crate::registry::manifest::{BundleDefinition, BundlesFile, ExtensionManifest, ManifestKind};
/// Error type for registry operations.
@@ -64,6 +65,69 @@ pub struct RegistryCatalog {
}
impl RegistryCatalog {
/// Find the `registry/` directory by searching relative to cwd, the executable,
/// and `CARGO_MANIFEST_DIR`. Returns `None` if the directory cannot be found
/// (non-fatal at startup).
pub fn find_dir() -> Option<PathBuf> {
// Try relative to current directory (for dev usage)
if let Ok(cwd) = std::env::current_dir() {
let candidate = cwd.join("registry");
if candidate.is_dir() {
return Some(candidate);
}
}
// Try relative to executable (covers installed binary, target/debug/, target/release/)
if let Ok(exe) = std::env::current_exe()
&& let Some(parent) = exe.parent()
{
// Walk up to 3 levels: exe dir, parent (target/release -> target), grandparent (-> repo root)
let mut dir = Some(parent);
for _ in 0..3 {
if let Some(d) = dir {
let candidate = d.join("registry");
if candidate.is_dir() {
return Some(candidate);
}
dir = d.parent();
}
}
}
// Try CARGO_MANIFEST_DIR (compile-time, works in dev builds)
let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
let candidate = manifest_dir.join("registry");
if candidate.is_dir() {
return Some(candidate);
}
None
}
/// Try to load from disk; if `registry/` cannot be found, fall back to
/// manifests embedded into the binary at compile time.
pub fn load_or_embedded() -> Result<Self, RegistryError> {
if let Some(dir) = Self::find_dir() {
return Self::load(&dir);
}
// Fall back to embedded catalog
let manifests = embedded::load_embedded();
let bundles = embedded::load_embedded_bundles();
tracing::info!(
"Loaded embedded registry catalog ({} extensions, {} bundles)",
manifests.len(),
bundles.len()
);
Ok(Self {
manifests,
bundles,
root: PathBuf::new(),
})
}
/// Load the catalog from a registry directory.
///
/// Expects the structure:
@@ -577,4 +641,12 @@ mod tests {
let result = RegistryCatalog::load(Path::new("/nonexistent/path"));
assert!(result.is_err());
}
#[test]
fn test_load_or_embedded_succeeds() {
// Should always succeed: either finds registry/ on disk or falls back to embedded
let catalog = RegistryCatalog::load_or_embedded().unwrap();
// At minimum, the embedded catalog from the repo should have entries
assert!(!catalog.all().is_empty() || !catalog.bundle_names().is_empty());
}
}
src/registry/embedded.rs
+97
View File
@@ -0,0 +1,97 @@
//! Embedded registry catalog compiled into the binary at build time.
//!
//! When IronClaw is distributed as a pre-built binary without a source tree,
//! the `registry/` directory is unavailable. This module provides the same
//! manifest data via `include_str!` from a JSON blob generated by `build.rs`.
use std::collections::HashMap;
use std::sync::OnceLock;
use crate::registry::manifest::{BundleDefinition, BundlesFile, ExtensionManifest};
/// Raw JSON generated by build.rs from `registry/{tools,channels}/*.json` and `_bundles.json`.
const EMBEDDED_CATALOG: &str = include_str!(concat!(env!("OUT_DIR"), "/embedded_catalog.json"));
/// Intermediate deserialization shape matching the build.rs output.
#[derive(serde::Deserialize)]
struct EmbeddedCatalogRaw {
#[serde(default)]
tools: Vec<ExtensionManifest>,
#[serde(default)]
channels: Vec<ExtensionManifest>,
#[serde(default)]
bundles: BundlesFile,
}
/// Parsed catalog cached across calls.
struct ParsedCatalog {
manifests: HashMap<String, ExtensionManifest>,
bundles: HashMap<String, BundleDefinition>,
}
fn parsed_catalog() -> &'static ParsedCatalog {
static CACHE: OnceLock<ParsedCatalog> = OnceLock::new();
CACHE.get_or_init(|| {
let raw: EmbeddedCatalogRaw = match serde_json::from_str(EMBEDDED_CATALOG) {
Ok(v) => v,
Err(e) => {
tracing::warn!("Failed to parse embedded catalog: {}", e);
return ParsedCatalog {
manifests: HashMap::new(),
bundles: HashMap::new(),
};
}
};
let mut manifests = HashMap::new();
for m in raw.tools {
let key = format!("tools/{}", m.name);
manifests.insert(key, m);
}
for m in raw.channels {
let key = format!("channels/{}", m.name);
manifests.insert(key, m);
}
ParsedCatalog {
manifests,
bundles: raw.bundles.bundles,
}
})
}
/// Load all embedded extension manifests, keyed by `"tools/<name>"` or `"channels/<name>"`.
pub fn load_embedded() -> HashMap<String, ExtensionManifest> {
parsed_catalog().manifests.clone()
}
/// Load embedded bundle definitions.
pub fn load_embedded_bundles() -> HashMap<String, BundleDefinition> {
parsed_catalog().bundles.clone()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_load_embedded_parses() {
let manifests = load_embedded();
// Should have at least the manifests from registry/ if built from the repo
// (empty is also valid for minimal builds without registry/)
assert!(
manifests.is_empty() || manifests.contains_key("tools/github"),
"Expected either empty catalog or github tool, got {} entries",
manifests.len()
);
}
#[test]
fn test_load_embedded_bundles_parses() {
let bundles = load_embedded_bundles();
assert!(
bundles.is_empty() || bundles.contains_key("default"),
"Expected either empty bundles or 'default' bundle"
);
}
}
src/registry/installer.rs
+325 -61
View File
@@ -137,6 +137,10 @@ impl RegistryInstaller {
}
/// Download and install a pre-built artifact.
///
/// Supports two formats:
/// - **tar.gz bundle**: Contains `{name}.wasm` + `{name}.capabilities.json`
/// - **bare .wasm file**: Just the WASM binary (capabilities fetched separately if available)
pub async fn install_from_artifact(
&self,
manifest: &ExtensionManifest,
@@ -156,13 +160,6 @@ impl RegistryInstaller {
))
})?;
let expected_sha = artifact.sha256.as_ref().ok_or_else(|| {
RegistryError::ExtensionNotFound(format!(
"No SHA256 hash for '{}'. Cannot verify download.",
manifest.name
))
})?;
let target_dir = match manifest.kind {
ManifestKind::Tool => &self.tools_dir,
ManifestKind::Channel => &self.channels_dir,
@@ -186,75 +183,90 @@ impl RegistryInstaller {
"Downloading {} '{}'...",
manifest.kind, manifest.display_name
);
let response = reqwest::get(url)
.await
.map_err(|e| RegistryError::DownloadFailed {
url: url.clone(),
reason: format!("request failed: {}", e),
})?;
let bytes = download_artifact(url).await?;
let response = response
.error_for_status()
.map_err(|e| RegistryError::DownloadFailed {
url: url.clone(),
reason: e.to_string(),
})?;
let bytes = response
.bytes()
.await
.map_err(|e| RegistryError::DownloadFailed {
url: url.clone(),
reason: format!("failed to read body: {}", e),
})?;
// Verify SHA256
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(&bytes);
let actual_sha = format!("{:x}", hasher.finalize());
if actual_sha != *expected_sha {
return Err(RegistryError::DownloadFailed {
url: url.clone(),
reason: format!(
"SHA256 mismatch: expected {}, got {}",
expected_sha, actual_sha
),
});
// Verify SHA256 if provided, warn otherwise
if let Some(expected_sha) = &artifact.sha256 {
verify_sha256(&bytes, expected_sha, url)?;
} else {
println!(
"WARNING: No SHA256 checksum for '{}'; download is not cryptographically verified.",
manifest.name
);
}
// Write file
fs::write(&target_wasm, &bytes)
.await
.map_err(RegistryError::Io)?;
// Copy capabilities from source dir (still needed even for pre-built artifacts).
// NOTE: This requires the source tree to be present. When pre-built artifact
// distribution is implemented, capabilities should be bundled with the artifact
// or fetched from a separate URL.
let caps_source = self
.repo_root
.join(&manifest.source.dir)
.join(&manifest.source.capabilities);
let target_caps = target_dir.join(format!("{}.capabilities.json", manifest.name));
let has_capabilities = if caps_source.exists() {
fs::copy(&caps_source, &target_caps)
// Detect format and extract
let has_capabilities = if is_gzip(&bytes) {
// tar.gz bundle: extract {name}.wasm and {name}.capabilities.json
let extracted =
extract_tar_gz(&bytes, &manifest.name, &target_wasm, &target_caps, url)?;
extracted.has_capabilities
} else {
// Bare WASM file
fs::write(&target_wasm, &bytes)
.await
.map_err(RegistryError::Io)?;
true
} else {
false
// Try to get capabilities from:
// 1. Separate capabilities_url in the artifact
// 2. Source tree (legacy, requires repo)
if let Some(ref caps_url) = artifact.capabilities_url {
const MAX_CAPS_SIZE: usize = 1024 * 1024; // 1 MB
match download_artifact(caps_url).await {
Ok(caps_bytes) if caps_bytes.len() <= MAX_CAPS_SIZE => {
fs::write(&target_caps, &caps_bytes)
.await
.map_err(RegistryError::Io)?;
true
}
Ok(caps_bytes) => {
tracing::warn!(
"Capabilities file too large ({} bytes, max {}), skipping",
caps_bytes.len(),
MAX_CAPS_SIZE
);
false
}
Err(e) => {
tracing::warn!("Failed to download capabilities from {}: {}", caps_url, e);
false
}
}
} else {
// Legacy fallback: try source tree
let caps_source = self
.repo_root
.join(&manifest.source.dir)
.join(&manifest.source.capabilities);
if caps_source.exists() {
fs::copy(&caps_source, &target_caps)
.await
.map_err(RegistryError::Io)?;
true
} else {
false
}
}
};
println!(" Installed to {}", target_wasm.display());
let mut warnings = Vec::new();
if !has_capabilities {
warnings.push(format!(
"No capabilities file found for '{}'. Auth and hooks may not work.",
manifest.name
));
}
Ok(InstallOutcome {
name: manifest.name.clone(),
kind: manifest.kind,
wasm_path: target_wasm,
has_capabilities,
warnings: Vec::new(),
warnings,
})
}
@@ -399,6 +411,159 @@ async fn build_wasm_component(source_dir: &Path, crate_name: &str) -> anyhow::Re
)
}
/// Download an artifact from a URL.
async fn download_artifact(url: &str) -> Result<bytes::Bytes, RegistryError> {
let response = reqwest::get(url)
.await
.map_err(|e| RegistryError::DownloadFailed {
url: url.to_string(),
reason: format!("request failed: {}", e),
})?;
let response = response
.error_for_status()
.map_err(|e| RegistryError::DownloadFailed {
url: url.to_string(),
reason: e.to_string(),
})?;
response
.bytes()
.await
.map_err(|e| RegistryError::DownloadFailed {
url: url.to_string(),
reason: format!("failed to read body: {}", e),
})
}
/// Verify SHA256 of downloaded bytes.
fn verify_sha256(bytes: &[u8], expected: &str, url: &str) -> Result<(), RegistryError> {
use sha2::{Digest, Sha256};
let mut hasher = Sha256::new();
hasher.update(bytes);
let actual = format!("{:x}", hasher.finalize());
if actual != expected {
return Err(RegistryError::DownloadFailed {
url: url.to_string(),
reason: format!("SHA256 mismatch: expected {}, got {}", expected, actual),
});
}
Ok(())
}
/// Check if bytes start with gzip magic number (0x1f 0x8b).
fn is_gzip(bytes: &[u8]) -> bool {
bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b
}
/// Result of extracting a tar.gz bundle.
struct ExtractResult {
has_capabilities: bool,
}
/// Extract a tar.gz archive, looking for `{name}.wasm` and `{name}.capabilities.json`.
fn extract_tar_gz(
bytes: &[u8],
name: &str,
target_wasm: &Path,
target_caps: &Path,
url: &str,
) -> Result<ExtractResult, RegistryError> {
use flate2::read::GzDecoder;
use tar::Archive;
use std::io::Read as _;
let decoder = GzDecoder::new(bytes);
let mut archive = Archive::new(decoder);
// Defense-in-depth: do not preserve permissions or extended attributes
archive.set_preserve_permissions(false);
#[cfg(any(unix, target_os = "redox"))]
archive.set_unpack_xattrs(false);
// 100 MB cap on decompressed entry size to prevent decompression bombs
const MAX_ENTRY_SIZE: u64 = 100 * 1024 * 1024;
let wasm_filename = format!("{}.wasm", name);
let caps_filename = format!("{}.capabilities.json", name);
let mut found_wasm = false;
let mut found_caps = false;
let entries = archive
.entries()
.map_err(|e| RegistryError::DownloadFailed {
url: url.to_string(),
reason: format!("failed to read tar.gz entries: {}", e),
})?;
for entry in entries {
let mut entry = entry.map_err(|e| RegistryError::DownloadFailed {
url: url.to_string(),
reason: format!("failed to read tar.gz entry: {}", e),
})?;
if entry.size() > MAX_ENTRY_SIZE {
return Err(RegistryError::DownloadFailed {
url: url.to_string(),
reason: format!(
"archive entry too large ({} bytes, max {} bytes)",
entry.size(),
MAX_ENTRY_SIZE
),
});
}
let entry_path = entry
.path()
.map_err(|e| RegistryError::DownloadFailed {
url: url.to_string(),
reason: format!("invalid path in tar.gz: {}", e),
})?
.to_path_buf();
// Match by filename (ignoring any directory prefix in the archive)
let filename = entry_path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
if filename == wasm_filename {
let mut data = Vec::with_capacity(entry.size() as usize);
std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data)
.map_err(|e| RegistryError::DownloadFailed {
url: url.to_string(),
reason: format!("failed to read {} from archive: {}", wasm_filename, e),
})?;
std::fs::write(target_wasm, &data).map_err(RegistryError::Io)?;
found_wasm = true;
} else if filename == caps_filename {
let mut data = Vec::with_capacity(entry.size() as usize);
std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data)
.map_err(|e| RegistryError::DownloadFailed {
url: url.to_string(),
reason: format!("failed to read {} from archive: {}", caps_filename, e),
})?;
std::fs::write(target_caps, &data).map_err(RegistryError::Io)?;
found_caps = true;
}
}
if !found_wasm {
return Err(RegistryError::DownloadFailed {
url: url.to_string(),
reason: format!(
"tar.gz archive does not contain '{}'. Archive may be malformed.",
wasm_filename
),
});
}
Ok(ExtractResult {
has_capabilities: found_caps,
})
}
#[cfg(test)]
mod tests {
use super::*;
@@ -412,4 +577,103 @@ mod tests {
);
assert_eq!(installer.repo_root, PathBuf::from("/repo"));
}
#[test]
fn test_is_gzip() {
assert!(is_gzip(&[0x1f, 0x8b, 0x08]));
assert!(!is_gzip(&[0x00, 0x61, 0x73, 0x6d])); // WASM magic
assert!(!is_gzip(&[0x1f])); // Too short
assert!(!is_gzip(&[]));
}
#[test]
fn test_verify_sha256_valid() {
use sha2::{Digest, Sha256};
let data = b"hello world";
let mut hasher = Sha256::new();
hasher.update(data);
let hash = format!("{:x}", hasher.finalize());
assert!(verify_sha256(data, &hash, "test://url").is_ok());
}
#[test]
fn test_verify_sha256_invalid() {
assert!(verify_sha256(b"data", "0000", "test://url").is_err());
}
#[test]
fn test_extract_tar_gz() {
use flate2::Compression;
use flate2::write::GzEncoder;
use tar::Builder;
// Create a tar.gz in memory with test.wasm and test.capabilities.json
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
{
let mut builder = Builder::new(&mut encoder);
let wasm_data = b"\0asm\x01\x00\x00\x00";
let mut header = tar::Header::new_gnu();
header.set_size(wasm_data.len() as u64);
header.set_cksum();
builder
.append_data(&mut header, "test.wasm", &wasm_data[..])
.unwrap();
let caps_data = br#"{"auth":null}"#;
let mut header = tar::Header::new_gnu();
header.set_size(caps_data.len() as u64);
header.set_cksum();
builder
.append_data(&mut header, "test.capabilities.json", &caps_data[..])
.unwrap();
builder.finish().unwrap();
}
let gz_bytes = encoder.finish().unwrap();
let tmp = tempfile::tempdir().unwrap();
let wasm_path = tmp.path().join("test.wasm");
let caps_path = tmp.path().join("test.capabilities.json");
let result =
extract_tar_gz(&gz_bytes, "test", &wasm_path, &caps_path, "test://url").unwrap();
assert!(wasm_path.exists());
assert!(caps_path.exists());
assert!(result.has_capabilities);
}
#[test]
fn test_extract_tar_gz_missing_wasm() {
use flate2::Compression;
use flate2::write::GzEncoder;
use tar::Builder;
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
{
let mut builder = Builder::new(&mut encoder);
let data = b"not a wasm file";
let mut header = tar::Header::new_gnu();
header.set_size(data.len() as u64);
header.set_cksum();
builder
.append_data(&mut header, "wrong.wasm", &data[..])
.unwrap();
builder.finish().unwrap();
}
let gz_bytes = encoder.finish().unwrap();
let tmp = tempfile::tempdir().unwrap();
let result = extract_tar_gz(
&gz_bytes,
"test",
&tmp.path().join("test.wasm"),
&tmp.path().join("test.capabilities.json"),
"test://url",
);
assert!(result.is_err());
}
}
src/registry/manifest.rs
+27 -5
View File
@@ -88,10 +88,17 @@ pub struct SourceSpec {
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ArtifactSpec {
/// Download URL (null until release).
/// Can point to a `.wasm` file or a `.tar.gz` bundle containing both
/// `{name}.wasm` and `{name}.capabilities.json`.
pub url: Option<String>,
/// Hex SHA256 of the WASM binary (null until release).
/// Hex SHA256 of the downloaded artifact (null until release).
pub sha256: Option<String>,
/// Optional separate download URL for the capabilities file.
/// Only needed when `url` points to a bare `.wasm` file instead of a bundle.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub capabilities_url: Option<String>,
}
/// Summary of authentication requirements extracted from capabilities.
@@ -138,7 +145,7 @@ pub struct BundleDefinition {
}
/// Top-level structure of `_bundles.json`.
#[derive(Debug, Clone, Serialize, Deserialize)]
#[derive(Debug, Clone, Default, Serialize, Deserialize)]
pub struct BundlesFile {
pub bundles: std::collections::HashMap<String, BundleDefinition>,
}
@@ -147,9 +154,24 @@ impl ExtensionManifest {
/// Convert this manifest into a [`RegistryEntry`] for use with the in-chat
/// extension discovery system.
pub fn to_registry_entry(&self) -> RegistryEntry {
let source = ExtensionSource::WasmBuildable {
repo_url: self.source.dir.clone(),
build_dir: Some(self.source.dir.clone()),
// Prefer pre-built artifact download when a URL is available
let source = if let Some(artifact) = self.artifacts.get("wasm32-wasip2") {
if let Some(ref url) = artifact.url {
ExtensionSource::WasmDownload {
wasm_url: url.clone(),
capabilities_url: artifact.capabilities_url.clone(),
}
} else {
ExtensionSource::WasmBuildable {
repo_url: self.source.dir.clone(),
build_dir: Some(self.source.dir.clone()),
}
}
} else {
ExtensionSource::WasmBuildable {
repo_url: self.source.dir.clone(),
build_dir: Some(self.source.dir.clone()),
}
};
let auth_hint = match self.auth_summary.as_ref().and_then(|a| a.method.as_deref()) {
src/registry/mod.rs
+1
View File
@@ -12,6 +12,7 @@
//! ```
pub mod catalog;
pub mod embedded;
pub mod installer;
pub mod manifest;
src/secrets/mod.rs
+1 -2
View File
@@ -74,5 +74,4 @@ pub use types::{
SecretError, SecretRef,
};
#[cfg(test)]
pub use store::testing::InMemorySecretsStore;
pub use store::in_memory::InMemorySecretsStore;
src/secrets/store.rs
+5 -4
View File
@@ -635,9 +635,10 @@ fn libsql_row_to_secret(row: &libsql::Row) -> Result<Secret, SecretError> {
})
}
/// In-memory implementation for testing.
#[cfg(test)]
pub mod testing {
/// In-memory secrets store. Used for testing and as a fallback when no
/// persistent secrets backend is configured (extension listing/install still
/// works, but stored secrets won't survive a restart).
pub mod in_memory {
use std::collections::HashMap;
use std::sync::Arc;
@@ -794,7 +795,7 @@ mod tests {
use crate::secrets::crypto::SecretsCrypto;
use crate::secrets::store::SecretsStore;
use crate::secrets::store::testing::InMemorySecretsStore;
use crate::secrets::store::in_memory::InMemorySecretsStore;
use crate::secrets::types::CreateSecretParams;
fn test_store() -> InMemorySecretsStore {
src/settings.rs
+5 -3
View File
@@ -1273,9 +1273,11 @@ mod tests {
let from_db = Settings::from_db_map(&db_map);
// Step 1 of the new wizard run: user enters a NEW database_url
let mut step1_settings = Settings::default();
step1_settings.database_backend = Some("postgres".to_string());
step1_settings.database_url = Some("postgres://new-host/ironclaw".to_string());
let step1_settings = Settings {
database_backend: Some("postgres".to_string()),
database_url: Some("postgres://new-host/ironclaw".to_string()),
..Settings::default()
};
// Wizard flow: load DB → merge_from(step1_overrides)
let mut current = step1_settings.clone();
src/setup/wizard.rs
+3 -33
View File
@@ -2566,40 +2566,10 @@ fn build_channel_options(discovered: &[(String, ChannelCapabilitiesFile)]) -> Ve
names
}
/// Try to load the registry catalog. Returns None if the registry directory
/// cannot be found (e.g. running from an installed binary without the repo).
/// Try to load the registry catalog. Falls back to embedded manifests when
/// the `registry/` directory cannot be found (e.g. running from an installed binary).
fn load_registry_catalog() -> Option<crate::registry::catalog::RegistryCatalog> {
// Try relative to current directory (dev usage)
let cwd = std::env::current_dir().ok()?;
let candidate = cwd.join("registry");
if candidate.is_dir() {
return crate::registry::catalog::RegistryCatalog::load(&candidate).ok();
}
// Try relative to executable
if let Ok(exe) = std::env::current_exe()
&& let Some(parent) = exe.parent()
{
let candidate = parent.join("registry");
if candidate.is_dir() {
return crate::registry::catalog::RegistryCatalog::load(&candidate).ok();
}
if let Some(grandparent) = parent.parent() {
let candidate = grandparent.join("registry");
if candidate.is_dir() {
return crate::registry::catalog::RegistryCatalog::load(&candidate).ok();
}
}
}
// Try CARGO_MANIFEST_DIR (compile-time, works in dev builds)
let manifest_dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR"));
let candidate = manifest_dir.join("registry");
if candidate.is_dir() {
return crate::registry::catalog::RegistryCatalog::load(&candidate).ok();
}
None
crate::registry::catalog::RegistryCatalog::load_or_embedded().ok()
}
/// Install selected channels from the registry that aren't already on disk
src/tools/builtin/extension_tools.rs
+1
View File
@@ -596,6 +596,7 @@ mod tests {
None,
"test".to_string(),
None,
Vec::new(),
))
}
}