feat: embedded registry catalog and WASM bundle install pipeline (#283)

* feat: embedded registry catalog and WASM bundle install pipeline

Embed registry manifests at compile time so the extension catalog is
available without network access. Add tar.gz bundle support for WASM
extension downloads (tools and channels), a /api/extensions/registry
endpoint, CI job to build and publish WASM bundles on release, and
ephemeral in-memory secrets fallback so the extension manager works
even without a persistent secrets store.

Key changes:
- build.rs: collect registry/*.json into embedded_catalog.json at compile time
- src/registry/embedded.rs + catalog.rs: load embedded or on-disk catalog
- src/extensions/manager.rs: download_and_install_wasm handles tar.gz bundles,
  bare .wasm files, and separate capabilities downloads; wasm channel install
- src/channels/web/server.rs: /api/extensions/registry endpoint + no-cache headers
- src/app.rs: ephemeral InMemorySecretsStore fallback for extension manager
- registry/*.json: populate artifact download URLs for release bundles
- .github/workflows/release.yml: build-wasm-extensions CI job
- Simplified setup wizard and CLI registry commands

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix: address PR review — archive hardening, decompression bomb guard, test fix

- Add 100 MB decompressed entry size cap to tar.gz extraction in both
  manager.rs and installer.rs to prevent decompression bombs
- Add archive.set_preserve_permissions(false) and set_unpack_xattrs(false)
  for defense-in-depth against malicious archives
- Fix test assertion logic in catalog.rs (|| → || with correct negation)
- Replace silent tar fallback in CI with explicit if/else for capabilities
- Add warning when installing without SHA256 verification

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix: resolve clippy warning in settings.rs and enforce zero-warnings policy

Use struct initializer with ..Default::default() instead of field
reassignment. Update CLAUDE.md to codify zero clippy warnings policy —
all warnings must be fixed before committing, including pre-existing ones.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix: address PR review round 2 — build reliability, caps validation, naming

- build.rs: emit per-file rerun-if-changed for reliable content tracking;
  fix bundles fallback to match BundlesFile shape ({"bundles":{}})
- embedded.rs: parse catalog once via OnceLock instead of double-parsing
- manager.rs + installer.rs: add 1 MB size cap on capabilities_url downloads
  with proper error surfacing
- secrets/store.rs: rename misleading `pub mod testing` to `pub mod in_memory`
- server.rs: track installed extensions by (name, kind) tuple to avoid
  false positives across different extension kinds

Co-Authored-By: Claude Opus 4.6 <[email protected]>

---------

Co-authored-by: Claude Opus 4.6 <[email protected]>
This commit is contained in:
Illia Polosukhin
2026-02-21 05:43:28 +00:00
committed by GitHub
co-authored by Claude Opus 4.6
parent 3d4c647216
commit 436066415b
40 changed files with 1406 additions and 257 deletions
+283 -53
View File
@@ -75,9 +75,15 @@ impl ExtensionManager {
tunnel_url: Option<String>,
user_id: String,
store: Option<Arc<dyn crate::db::Database>>,
catalog_entries: Vec<RegistryEntry>,
) -> Self {
let registry = if catalog_entries.is_empty() {
ExtensionRegistry::new()
} else {
ExtensionRegistry::new_with_catalog(catalog_entries)
};
Self {
registry: ExtensionRegistry::new(),
registry,
discovery: OnlineDiscovery::new(),
mcp_session_manager,
mcp_clients: RwLock::new(HashMap::new()),
@@ -131,9 +137,14 @@ impl ExtensionManager {
url: Option<&str>,
kind_hint: Option<ExtensionKind>,
) -> Result<InstallResult, ExtensionError> {
tracing::info!(extension = %name, url = ?url, kind = ?kind_hint, "Installing extension");
// If we have a registry entry, use it
if let Some(entry) = self.registry.get(name).await {
return self.install_from_entry(&entry).await;
return self.install_from_entry(&entry).await.map_err(|e| {
tracing::error!(extension = %name, error = %e, "Extension install failed");
e
});
}
// If a URL was provided, determine kind and install
@@ -143,19 +154,21 @@ impl ExtensionManager {
ExtensionKind::McpServer => self.install_mcp_from_url(name, url).await,
ExtensionKind::WasmTool => self.install_wasm_tool_from_url(name, url).await,
ExtensionKind::WasmChannel => {
Err(ExtensionError::InstallFailed(
"WASM channel installation from URL not yet supported. \
Place the .wasm and .capabilities.json files in ~/.ironclaw/channels/ and restart."
.to_string(),
))
self.install_wasm_channel_from_url(name, url, None).await
}
};
}
.map_err(|e| {
tracing::error!(extension = %name, url = %url, error = %e, "Extension install from URL failed");
e
});
}
Err(ExtensionError::NotFound(format!(
let err = ExtensionError::NotFound(format!(
"'{}' not found in registry. Try searching with discover:true or provide a URL.",
name
)))
));
tracing::warn!(extension = %name, "Extension not found in registry");
Err(err)
}
/// Authenticate an installed extension.
@@ -433,16 +446,51 @@ impl ExtensionManager {
self.install_mcp_from_url(&entry.name, &url).await
}
ExtensionKind::WasmTool => match &entry.source {
ExtensionSource::WasmDownload { wasm_url, .. } => {
self.install_wasm_tool_from_url(&entry.name, wasm_url).await
ExtensionSource::WasmDownload {
wasm_url,
capabilities_url,
} => {
self.install_wasm_tool_from_url_with_caps(
&entry.name,
wasm_url,
capabilities_url.as_deref(),
)
.await
}
ExtensionSource::WasmBuildable { .. } => {
Err(ExtensionError::InstallFailed(format!(
"'{}' requires building from source. Run `ironclaw registry install {}` \
from the CLI (requires cargo-component).",
entry.name, entry.name
)))
}
_ => Err(ExtensionError::InstallFailed(
"WASM tool entry has no download URL".to_string(),
)),
},
ExtensionKind::WasmChannel => Err(ExtensionError::InstallFailed(
"WASM channel installation not yet supported via this flow".to_string(),
)),
ExtensionKind::WasmChannel => match &entry.source {
ExtensionSource::WasmDownload {
wasm_url,
capabilities_url,
} => {
self.install_wasm_channel_from_url(
&entry.name,
wasm_url,
capabilities_url.as_deref(),
)
.await
}
ExtensionSource::WasmBuildable { .. } => {
Err(ExtensionError::InstallFailed(format!(
"'{}' requires building from source. Run `ironclaw registry install {}` \
from the CLI (requires cargo-component).",
entry.name, entry.name
)))
}
_ => Err(ExtensionError::InstallFailed(
"WASM channel entry has no download URL".to_string(),
)),
},
}
}
@@ -482,6 +530,57 @@ impl ExtensionManager {
name: &str,
url: &str,
) -> Result<InstallResult, ExtensionError> {
self.install_wasm_tool_from_url_with_caps(name, url, None)
.await
}
async fn install_wasm_tool_from_url_with_caps(
&self,
name: &str,
url: &str,
capabilities_url: Option<&str>,
) -> Result<InstallResult, ExtensionError> {
self.download_and_install_wasm(name, url, capabilities_url, &self.wasm_tools_dir)
.await?;
Ok(InstallResult {
name: name.to_string(),
kind: ExtensionKind::WasmTool,
message: format!("WASM tool '{}' installed. Run activate to load it.", name),
})
}
async fn install_wasm_channel_from_url(
&self,
name: &str,
url: &str,
capabilities_url: Option<&str>,
) -> Result<InstallResult, ExtensionError> {
self.download_and_install_wasm(name, url, capabilities_url, &self.wasm_channels_dir)
.await?;
Ok(InstallResult {
name: name.to_string(),
kind: ExtensionKind::WasmChannel,
message: format!(
"WASM channel '{}' installed to {}. Restart to activate.",
name,
self.wasm_channels_dir.display()
),
})
}
/// Download a WASM extension (tool or channel) from URL and install to target directory.
///
/// Handles both tar.gz bundles (containing `.wasm` + `.capabilities.json`) and bare
/// `.wasm` files. Validates HTTPS, size limits, and file format.
async fn download_and_install_wasm(
&self,
name: &str,
url: &str,
capabilities_url: Option<&str>,
target_dir: &std::path::Path,
) -> Result<(), ExtensionError> {
// Require HTTPS to prevent downgrade attacks
if !url.starts_with("https://") {
return Err(ExtensionError::InstallFailed(
@@ -490,33 +589,41 @@ impl ExtensionManager {
}
// 50 MB cap to prevent disk-fill DoS
const MAX_WASM_SIZE: usize = 50 * 1024 * 1024;
const MAX_DOWNLOAD_SIZE: usize = 50 * 1024 * 1024;
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(60))
.build()
.map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?;
let response = client
.get(url)
.send()
.await
.map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?;
tracing::debug!(extension = %name, url = %url, "Downloading WASM extension");
let response = client.get(url).send().await.map_err(|e| {
tracing::error!(extension = %name, url = %url, error = %e, "Download request failed");
ExtensionError::DownloadFailed(e.to_string())
})?;
if !response.status().is_success() {
let status = response.status();
tracing::error!(
extension = %name,
url = %url,
status = %status,
"Download returned non-success HTTP status"
);
return Err(ExtensionError::DownloadFailed(format!(
"HTTP {}",
response.status()
"HTTP {} from {}",
status, url
)));
}
// Check Content-Length header before downloading the full body
if let Some(len) = response.content_length()
&& len as usize > MAX_WASM_SIZE
&& len as usize > MAX_DOWNLOAD_SIZE
{
return Err(ExtensionError::InstallFailed(format!(
"WASM binary too large ({} bytes, max {} bytes)",
len, MAX_WASM_SIZE
"Download too large ({} bytes, max {} bytes)",
len, MAX_DOWNLOAD_SIZE
)));
}
@@ -525,45 +632,164 @@ impl ExtensionManager {
.await
.map_err(|e| ExtensionError::DownloadFailed(e.to_string()))?;
if bytes.len() > MAX_WASM_SIZE {
if bytes.len() > MAX_DOWNLOAD_SIZE {
return Err(ExtensionError::InstallFailed(format!(
"WASM binary too large ({} bytes, max {} bytes)",
"Download too large ({} bytes, max {} bytes)",
bytes.len(),
MAX_WASM_SIZE
MAX_DOWNLOAD_SIZE
)));
}
// Basic WASM magic number check (\0asm)
if bytes.len() < 4 || &bytes[..4] != b"\0asm" {
return Err(ExtensionError::InstallFailed(
"Downloaded file is not a valid WASM binary (bad magic number)".to_string(),
));
// Ensure target directory exists
tokio::fs::create_dir_all(target_dir)
.await
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
let wasm_path = target_dir.join(format!("{}.wasm", name));
let caps_path = target_dir.join(format!("{}.capabilities.json", name));
// Detect format: gzip (tar.gz bundle) or bare WASM
if bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b {
// tar.gz bundle: extract {name}.wasm and {name}.capabilities.json
self.extract_wasm_tar_gz(name, &bytes, &wasm_path, &caps_path)?;
} else {
// Bare WASM file: validate magic number
if bytes.len() < 4 || &bytes[..4] != b"\0asm" {
return Err(ExtensionError::InstallFailed(
"Downloaded file is not a valid WASM binary (bad magic number)".to_string(),
));
}
tokio::fs::write(&wasm_path, &bytes)
.await
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
// Download capabilities separately if URL provided
if let Some(caps_url) = capabilities_url {
const MAX_CAPS_SIZE: usize = 1024 * 1024; // 1 MB
match client.get(caps_url).send().await {
Ok(resp) if resp.status().is_success() => match resp.bytes().await {
Ok(caps_bytes) if caps_bytes.len() <= MAX_CAPS_SIZE => {
if let Err(e) = tokio::fs::write(&caps_path, &caps_bytes).await {
tracing::warn!(
"Failed to write capabilities for '{}': {}",
name,
e
);
}
}
Ok(caps_bytes) => {
tracing::warn!(
"Capabilities file for '{}' too large ({} bytes, max {})",
name,
caps_bytes.len(),
MAX_CAPS_SIZE
);
}
Err(e) => {
tracing::warn!("Failed to download capabilities for '{}': {}", name, e);
}
},
_ => {
tracing::warn!(
"Failed to download capabilities for '{}' from {}",
name,
caps_url
);
}
}
}
}
// Ensure tools directory exists
tokio::fs::create_dir_all(&self.wasm_tools_dir)
.await
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
// Write the WASM file
let wasm_path = self.wasm_tools_dir.join(format!("{}.wasm", name));
tokio::fs::write(&wasm_path, &bytes)
.await
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
tracing::info!(
"Installed WASM tool '{}' ({} bytes) from {} to {}",
"Installed WASM extension '{}' from {} to {}",
name,
bytes.len(),
url,
wasm_path.display()
);
Ok(InstallResult {
name: name.to_string(),
kind: ExtensionKind::WasmTool,
message: format!("WASM tool '{}' installed. Run activate to load it.", name),
})
Ok(())
}
/// Extract a tar.gz bundle into the WASM tools directory.
fn extract_wasm_tar_gz(
&self,
name: &str,
bytes: &[u8],
target_wasm: &std::path::Path,
target_caps: &std::path::Path,
) -> Result<(), ExtensionError> {
use flate2::read::GzDecoder;
use tar::Archive;
use std::io::Read as _;
let decoder = GzDecoder::new(bytes);
let mut archive = Archive::new(decoder);
// Defense-in-depth: do not preserve permissions or extended attributes
archive.set_preserve_permissions(false);
#[cfg(any(unix, target_os = "redox"))]
archive.set_unpack_xattrs(false);
// 100 MB cap on decompressed entry size to prevent decompression bombs
const MAX_ENTRY_SIZE: u64 = 100 * 1024 * 1024;
let wasm_filename = format!("{}.wasm", name);
let caps_filename = format!("{}.capabilities.json", name);
let mut found_wasm = false;
let entries = archive
.entries()
.map_err(|e| ExtensionError::InstallFailed(format!("Bad tar.gz archive: {}", e)))?;
for entry in entries {
let mut entry = entry
.map_err(|e| ExtensionError::InstallFailed(format!("Bad tar.gz entry: {}", e)))?;
if entry.size() > MAX_ENTRY_SIZE {
return Err(ExtensionError::InstallFailed(format!(
"Archive entry too large ({} bytes, max {} bytes)",
entry.size(),
MAX_ENTRY_SIZE
)));
}
let entry_path = entry
.path()
.map_err(|e| {
ExtensionError::InstallFailed(format!("Invalid path in tar.gz: {}", e))
})?
.to_path_buf();
let filename = entry_path
.file_name()
.and_then(|n| n.to_str())
.unwrap_or("");
if filename == wasm_filename {
let mut data = Vec::with_capacity(entry.size() as usize);
std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data)
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
std::fs::write(target_wasm, &data)
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
found_wasm = true;
} else if filename == caps_filename {
let mut data = Vec::with_capacity(entry.size() as usize);
std::io::Read::read_to_end(&mut entry.by_ref().take(MAX_ENTRY_SIZE), &mut data)
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
std::fs::write(target_caps, &data)
.map_err(|e| ExtensionError::InstallFailed(e.to_string()))?;
}
}
if !found_wasm {
return Err(ExtensionError::InstallFailed(format!(
"tar.gz archive does not contain '{}'",
wasm_filename
)));
}
Ok(())
}
async fn auth_mcp(
@@ -1074,7 +1300,7 @@ impl ExtensionManager {
/// Infer the extension kind from a URL.
fn infer_kind_from_url(url: &str) -> ExtensionKind {
if url.ends_with(".wasm") {
if url.ends_with(".wasm") || url.ends_with(".tar.gz") {
ExtensionKind::WasmTool
} else {
ExtensionKind::McpServer
@@ -1092,6 +1318,10 @@ mod tests {
infer_kind_from_url("https://example.com/tool.wasm"),
ExtensionKind::WasmTool
);
assert_eq!(
infer_kind_from_url("https://example.com/tool-wasm32-wasip2.tar.gz"),
ExtensionKind::WasmTool
);
assert_eq!(
infer_kind_from_url("https://mcp.notion.com"),
ExtensionKind::McpServer
+92
View File
@@ -26,6 +26,26 @@ impl ExtensionRegistry {
}
}
/// Create a new registry merging builtin entries with catalog-provided entries.
///
/// Deduplicates by `(name, kind)` pair -- a builtin MCP "slack" and a registry
/// WASM "slack" can coexist since they're different kinds.
pub fn new_with_catalog(catalog_entries: Vec<RegistryEntry>) -> Self {
let mut entries = builtin_entries();
for entry in catalog_entries {
if !entries
.iter()
.any(|e| e.name == entry.name && e.kind == entry.kind)
{
entries.push(entry);
}
}
Self {
entries,
discovery_cache: RwLock::new(Vec::new()),
}
}
/// Search the registry by query string. Returns results sorted by relevance.
///
/// Splits the query into lowercase tokens and scores each entry by matches
@@ -542,4 +562,76 @@ mod tests {
let results = registry.search("dup").await;
assert_eq!(results.len(), 1, "Should not duplicate cached entries");
}
#[tokio::test]
async fn test_new_with_catalog() {
let catalog_entries = vec![
RegistryEntry {
name: "telegram".to_string(),
display_name: "Telegram".to_string(),
kind: ExtensionKind::WasmChannel,
description: "Telegram Bot API channel".to_string(),
keywords: vec!["messaging".into(), "bot".into()],
source: ExtensionSource::WasmBuildable {
repo_url: "channels-src/telegram".to_string(),
build_dir: Some("channels-src/telegram".to_string()),
},
auth_hint: AuthHint::CapabilitiesAuth,
},
// This shares a name with a builtin but has a different kind, so both should appear
RegistryEntry {
name: "slack".to_string(),
display_name: "Slack WASM".to_string(),
kind: ExtensionKind::WasmTool,
description: "Slack WASM tool".to_string(),
keywords: vec!["messaging".into()],
source: ExtensionSource::WasmBuildable {
repo_url: "tools-src/slack".to_string(),
build_dir: Some("tools-src/slack".to_string()),
},
auth_hint: AuthHint::CapabilitiesAuth,
},
];
let registry = ExtensionRegistry::new_with_catalog(catalog_entries);
// Should find the new telegram entry
let results = registry.search("telegram").await;
assert!(!results.is_empty(), "Should find telegram from catalog");
assert_eq!(results[0].entry.name, "telegram");
// Should have both builtin MCP slack and catalog WASM slack
let results = registry.search("slack").await;
let slack_mcp = results
.iter()
.any(|r| r.entry.name == "slack" && r.entry.kind == ExtensionKind::McpServer);
let slack_wasm = results
.iter()
.any(|r| r.entry.name == "slack" && r.entry.kind == ExtensionKind::WasmTool);
assert!(slack_mcp, "Should have builtin MCP slack");
assert!(slack_wasm, "Should have catalog WASM slack");
}
#[tokio::test]
async fn test_new_with_catalog_dedup_same_kind() {
// A catalog entry with same name AND kind as a builtin should be skipped
let catalog_entries = vec![RegistryEntry {
name: "slack".to_string(),
display_name: "Slack Override".to_string(),
kind: ExtensionKind::McpServer, // same kind as builtin
description: "Should be skipped".to_string(),
keywords: vec![],
source: ExtensionSource::McpUrl {
url: "https://other.slack.com".to_string(),
},
auth_hint: AuthHint::Dcr,
}];
let registry = ExtensionRegistry::new_with_catalog(catalog_entries);
let entry = registry.get("slack").await;
assert!(entry.is_some());
// Should still be the builtin, not the override
assert_eq!(entry.unwrap().display_name, "Slack");
}
}