gitops/optimclaw
1
0
Fork 0
mirror of https://github.com/outbackdingo/optimclaw.git synced 2026-08-31 00:29:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: make onboarding installs prefer release artifacts with source fallback (#323)

Browse Source 
* fix: make onboarding installs prefer release artifacts with source fallback

* fix: harden extension fallback errors and surface setup warnings

* fix: validate registry artifacts and harden fallback errors

* fix: address review feedback on installer fallback

- Add upfront validate_manifest_install_inputs() in
  install_with_source_fallback so bad manifests fail fast without
  relying on inner methods to catch them
- Document ALLOWED_ARTIFACT_HOSTS as GitHub-only by design
- Document intentional url omission from DownloadFailed Display
- Add channel manifest validation tests (wrong prefix rejected,
  correct prefix accepted)

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix: require SHA256 checksum for artifact downloads

Reject artifact installs when the manifest has sha256: null instead of
warning and proceeding. This prevents installing unverified pre-built
binaries during onboarding. The check runs before downloading to avoid
wasting bandwidth.

Since InvalidManifest blocks source fallback, manifests with URLs but
no checksums will hard-fail rather than silently falling back to source
build — forcing the manifest to be fixed.

The release CI already computes SHA256 for each bundle; the manifests
just need to be populated with the actual values.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

* fix: enforce SHA256 checksums and auto-patch manifests in CI

- Fix cargo fmt on SHA256 check code
- Reorder release CI: build WASM extensions before binary so manifests
  can be patched with computed SHA256 before build.rs embeds them
- Add "Patch manifests with WASM checksums" step in build-local-artifacts
  that reads checksums.txt and updates registry JSON files before building
- Add update-registry-checksums job that commits patched manifests back
  to main after release, keeping the repo in sync with released artifacts

This closes the integrity gap where all manifests had sha256: null and
artifact downloads were unverified. The binary now embeds correct SHA256
values and the installer hard-rejects null checksums.

Co-Authored-By: Claude Opus 4.6 <[email protected]>

---------

Co-authored-by: Claude Opus 4.6 <[email protected]>
Co-authored-by: Bowen Wang <[email protected]>
This commit is contained in:
firat.sertgoz
2026-02-23 18:49:18 +00:00
committed by GitHub
co-authored by Claude Opus 4.6 Bowen Wang
parent cbf5c93578
commit 3e552e0e8e
5 changed files with 606 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/registry/catalog.rs
+34 -1
View File
@@ -27,9 +27,42 @@ pub enum RegistryError {
path: std::path::PathBuf,
},
#[error("Download failed for {url}: {reason}")]
// `url` is stored for programmatic access (logs, retries) but intentionally
// omitted from the Display message to avoid leaking internal artifact URLs
// to end users.
#[error("Artifact download failed: {reason}")]
DownloadFailed { url: String, reason: String },
#[error("Invalid extension manifest for '{name}' field '{field}': {reason}")]
InvalidManifest {
name: String,
field: &'static str,
reason: String,
},
#[error("Checksum verification failed: expected {expected_sha256}, got {actual_sha256}")]
ChecksumMismatch {
url: String,
expected_sha256: String,
actual_sha256: String,
},
#[error(
"Source fallback unavailable for '{name}' after artifact install failed. Retry artifact download or run from a repository checkout."
)]
SourceFallbackUnavailable {
name: String,
source_dir: PathBuf,
artifact_error: Box<RegistryError>,
},
#[error("Artifact install and source fallback both failed for '{name}'.")]
InstallFallbackFailed {
name: String,
artifact_error: Box<RegistryError>,
source_error: Box<RegistryError>,
},
#[error(
"Ambiguous name '{name}': exists as both {kind_a} and {kind_b}. Use '{prefix_a}/{name}' or '{prefix_b}/{name}'."
)]
src/registry/installer.rs
+476 -17
View File
@@ -1,12 +1,148 @@
//! Install extensions from the registry: build-from-source or download pre-built artifacts.
use std::path::{Path, PathBuf};
use std::net::IpAddr;
use std::path::{Component, Path, PathBuf};
use tokio::fs;
use crate::registry::catalog::RegistryError;
use crate::registry::manifest::{BundleDefinition, ExtensionManifest, ManifestKind};
// GitHub-only by design. New trusted hosts (e.g. a NEAR AI CDN) must be
// explicitly added here; unknown hosts fall back to source build with a
// warning rather than surfacing a clear "host not allowed" error.
const ALLOWED_ARTIFACT_HOSTS: &[&str] = &[
"github.com",
"objects.githubusercontent.com",
"github-releases.githubusercontent.com",
"raw.githubusercontent.com",
];
fn should_attempt_source_fallback(err: &RegistryError) -> bool {
!matches!(
err,
RegistryError::AlreadyInstalled { .. }
| RegistryError::ChecksumMismatch { .. }
| RegistryError::InvalidManifest { .. }
)
}
fn is_allowed_artifact_host(host: &str) -> bool {
ALLOWED_ARTIFACT_HOSTS
.iter()
.any(|allowed| host.eq_ignore_ascii_case(allowed))
|| host.ends_with(".githubusercontent.com")
}
fn validate_artifact_url(
manifest_name: &str,
field: &'static str,
url: &str,
) -> Result<(), RegistryError> {
let parsed = reqwest::Url::parse(url).map_err(|e| RegistryError::InvalidManifest {
name: manifest_name.to_string(),
field,
reason: format!("invalid URL: {}", e),
})?;
if parsed.scheme() != "https" {
return Err(RegistryError::InvalidManifest {
name: manifest_name.to_string(),
field,
reason: "URL must use https".to_string(),
});
}
let host = parsed
.host_str()
.ok_or_else(|| RegistryError::InvalidManifest {
name: manifest_name.to_string(),
field,
reason: "URL host is missing".to_string(),
})?;
if host.parse::<IpAddr>().is_ok() || !is_allowed_artifact_host(host) {
return Err(RegistryError::InvalidManifest {
name: manifest_name.to_string(),
field,
reason: format!("host '{}' is not allowed", host),
});
}
Ok(())
}
fn validate_manifest_install_inputs(manifest: &ExtensionManifest) -> Result<(), RegistryError> {
let is_valid_name = !manifest.name.is_empty()
&& manifest
.name
.chars()
.all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-' || c == '_');
if !is_valid_name {
return Err(RegistryError::InvalidManifest {
name: manifest.name.clone(),
field: "name",
reason: "name must contain only lowercase letters, digits, '-' or '_'".to_string(),
});
}
let expected_prefix = match manifest.kind {
ManifestKind::Tool => "tools-src/",
ManifestKind::Channel => "channels-src/",
};
if !manifest.source.dir.starts_with(expected_prefix) {
return Err(RegistryError::InvalidManifest {
name: manifest.name.clone(),
field: "source.dir",
reason: format!("must start with '{}'", expected_prefix),
});
}
let source_path = Path::new(&manifest.source.dir);
let has_unsafe_component = source_path.components().any(|component| {
matches!(
component,
Component::ParentDir | Component::RootDir | Component::Prefix(_) | Component::CurDir
)
});
if source_path.is_absolute() || has_unsafe_component {
return Err(RegistryError::InvalidManifest {
name: manifest.name.clone(),
field: "source.dir",
reason: "must be a safe relative path without traversal segments".to_string(),
});
}
let has_path_separator = manifest.source.capabilities.contains('/')
|| manifest.source.capabilities.contains('\\')
|| manifest.source.capabilities.contains("..");
if has_path_separator {
return Err(RegistryError::InvalidManifest {
name: manifest.name.clone(),
field: "source.capabilities",
reason: "must be a file name without path separators".to_string(),
});
}
Ok(())
}
fn download_failure_reason(error: &reqwest::Error) -> String {
if error.is_timeout() {
"request timed out".to_string()
} else if error.is_connect() {
"connection failed".to_string()
} else if error.is_request() {
"request failed".to_string()
} else {
"network error".to_string()
}
}
/// Result of installing a single extension from the registry.
#[derive(Debug)]
pub struct InstallOutcome {
@@ -57,6 +193,8 @@ impl RegistryInstaller {
manifest: &ExtensionManifest,
force: bool,
) -> Result<InstallOutcome, RegistryError> {
validate_manifest_install_inputs(manifest)?;
let source_dir = self.repo_root.join(&manifest.source.dir);
if !source_dir.exists() {
return Err(RegistryError::ManifestRead {
@@ -137,6 +275,67 @@ impl RegistryInstaller {
})
}
pub async fn install_with_source_fallback(
&self,
manifest: &ExtensionManifest,
force: bool,
) -> Result<InstallOutcome, RegistryError> {
// Validate upfront so we fail fast on bad manifests regardless of
// which install path runs, without relying on inner methods to
// catch it first.
validate_manifest_install_inputs(manifest)?;
let has_artifact = manifest
.artifacts
.get("wasm32-wasip2")
.and_then(|a| a.url.as_ref())
.is_some();
if !has_artifact {
return self.install_from_source(manifest, force).await;
}
let source_dir = self.repo_root.join(&manifest.source.dir);
match self.install_from_artifact(manifest, force).await {
Ok(outcome) => Ok(outcome),
Err(artifact_err) => {
if !should_attempt_source_fallback(&artifact_err) {
return Err(artifact_err);
}
if !source_dir.is_dir() {
return Err(RegistryError::SourceFallbackUnavailable {
name: manifest.name.clone(),
source_dir,
artifact_error: Box::new(artifact_err),
});
}
tracing::warn!(
extension = %manifest.name,
error = %artifact_err,
"Artifact install failed; falling back to build-from-source"
);
match self.install_from_source(manifest, force).await {
Ok(mut outcome) => {
outcome.warnings.push(format!(
"Artifact install failed ({}); installed via source fallback.",
artifact_err
));
Ok(outcome)
}
Err(source_err) => Err(RegistryError::InstallFallbackFailed {
name: manifest.name.clone(),
artifact_error: Box::new(artifact_err),
source_error: Box::new(source_err),
}),
}
}
}
}
/// Download and install a pre-built artifact.
///
/// Supports two formats:
@@ -147,6 +346,8 @@ impl RegistryInstaller {
manifest: &ExtensionManifest,
force: bool,
) -> Result<InstallOutcome, RegistryError> {
validate_manifest_install_inputs(manifest)?;
let artifact = manifest.artifacts.get("wasm32-wasip2").ok_or_else(|| {
RegistryError::ExtensionNotFound(format!(
"No wasm32-wasip2 artifact for '{}'",
@@ -161,6 +362,21 @@ impl RegistryInstaller {
))
})?;
validate_artifact_url(&manifest.name, "artifacts.wasm32-wasip2.url", url)?;
// Require SHA256 — refuse to install unverified binaries. Check before
// downloading to avoid wasting bandwidth on manifests that are missing
// checksums.
let expected_sha =
artifact
.sha256
.as_ref()
.ok_or_else(|| RegistryError::InvalidManifest {
name: manifest.name.clone(),
field: "artifacts.wasm32-wasip2.sha256",
reason: "sha256 is required for artifact downloads".to_string(),
})?;
let target_dir = match manifest.kind {
ManifestKind::Tool => &self.tools_dir,
ManifestKind::Channel => &self.channels_dir,
@@ -185,16 +401,7 @@ impl RegistryInstaller {
manifest.kind, manifest.display_name
);
let bytes = download_artifact(url).await?;
// Verify SHA256 if provided, warn otherwise
if let Some(expected_sha) = &artifact.sha256 {
verify_sha256(&bytes, expected_sha, url)?;
} else {
println!(
"WARNING: No SHA256 checksum for '{}'; download is not cryptographically verified.",
manifest.name
);
}
verify_sha256(&bytes, expected_sha, url)?;
let target_caps = target_dir.join(format!("{}.capabilities.json", manifest.name));
@@ -214,6 +421,11 @@ impl RegistryInstaller {
// 1. Separate capabilities_url in the artifact
// 2. Source tree (legacy, requires repo)
if let Some(ref caps_url) = artifact.capabilities_url {
validate_artifact_url(
&manifest.name,
"artifacts.wasm32-wasip2.capabilities_url",
caps_url,
)?;
const MAX_CAPS_SIZE: usize = 1024 * 1024; // 1 MB
match download_artifact(caps_url).await {
Ok(caps_bytes) if caps_bytes.len() <= MAX_CAPS_SIZE => {
@@ -360,14 +572,18 @@ async fn download_artifact(url: &str) -> Result<bytes::Bytes, RegistryError> {
.await
.map_err(|e| RegistryError::DownloadFailed {
url: url.to_string(),
reason: format!("request failed: {}", e),
reason: download_failure_reason(&e),
})?;
let response = response
.error_for_status()
.map_err(|e| RegistryError::DownloadFailed {
url: url.to_string(),
reason: e.to_string(),
reason: format!(
"http status {}",
e.status()
.map_or("unknown".to_string(), |status| status.as_u16().to_string())
),
})?;
response
@@ -375,7 +591,7 @@ async fn download_artifact(url: &str) -> Result<bytes::Bytes, RegistryError> {
.await
.map_err(|e| RegistryError::DownloadFailed {
url: url.to_string(),
reason: format!("failed to read body: {}", e),
reason: format!("failed to read response body: {}", e),
})
}
@@ -387,9 +603,10 @@ fn verify_sha256(bytes: &[u8], expected: &str, url: &str) -> Result<(), Registry
let actual = format!("{:x}", hasher.finalize());
if actual != expected {
return Err(RegistryError::DownloadFailed {
return Err(RegistryError::ChecksumMismatch {
url: url.to_string(),
reason: format!("SHA256 mismatch: expected {}, got {}", expected, actual),
expected_sha256: expected.to_string(),
actual_sha256: actual,
});
}
Ok(())
@@ -510,6 +727,55 @@ fn extract_tar_gz(
#[cfg(test)]
mod tests {
use super::*;
use std::collections::HashMap;
use crate::registry::manifest::{ArtifactSpec, SourceSpec};
fn test_manifest(
name: &str,
source_dir: &str,
artifact_url: Option<String>,
sha256: Option<&str>,
) -> ExtensionManifest {
test_manifest_with_kind(name, source_dir, artifact_url, sha256, ManifestKind::Tool)
}
fn test_manifest_with_kind(
name: &str,
source_dir: &str,
artifact_url: Option<String>,
sha256: Option<&str>,
kind: ManifestKind,
) -> ExtensionManifest {
let mut artifacts = HashMap::new();
if artifact_url.is_some() || sha256.is_some() {
artifacts.insert(
"wasm32-wasip2".to_string(),
ArtifactSpec {
url: artifact_url,
sha256: sha256.map(ToString::to_string),
capabilities_url: None,
},
);
}
ExtensionManifest {
name: name.to_string(),
display_name: name.to_string(),
kind,
version: "0.1.0".to_string(),
description: "test manifest".to_string(),
keywords: Vec::new(),
source: SourceSpec {
dir: source_dir.to_string(),
capabilities: format!("{}.capabilities.json", name),
crate_name: name.to_string(),
},
artifacts,
auth_summary: None,
tags: Vec::new(),
}
}
#[test]
fn test_installer_creation() {
@@ -541,7 +807,140 @@ mod tests {
#[test]
fn test_verify_sha256_invalid() {
assert!(verify_sha256(b"data", "0000", "test://url").is_err());
let err = verify_sha256(b"data", "0000", "test://url").expect_err("checksum mismatch");
assert!(matches!(err, RegistryError::ChecksumMismatch { .. }));
}
#[tokio::test]
async fn test_install_from_source_rejects_path_traversal_name() {
let temp = tempfile::tempdir().expect("tempdir");
let installer = RegistryInstaller::new(
temp.path().to_path_buf(),
temp.path().join("tools"),
temp.path().join("channels"),
);
let manifest = test_manifest("../evil", "tools-src/evil", None, None);
let result = installer.install_from_source(&manifest, false).await;
match result {
Err(RegistryError::InvalidManifest { field, .. }) => {
assert_eq!(field, "name");
}
other => panic!("unexpected result: {:?}", other),
}
}
#[tokio::test]
async fn test_install_from_artifact_rejects_non_https_url() {
let temp = tempfile::tempdir().expect("tempdir");
let installer = RegistryInstaller::new(
temp.path().to_path_buf(),
temp.path().join("tools"),
temp.path().join("channels"),
);
let manifest = test_manifest(
"demo",
"tools-src/demo",
Some(
"http://github.com/nearai/ironclaw/releases/latest/download/demo.wasm".to_string(),
),
None,
);
let result = installer.install_from_artifact(&manifest, false).await;
match result {
Err(RegistryError::InvalidManifest { field, .. }) => {
assert_eq!(field, "artifacts.wasm32-wasip2.url");
}
other => panic!("unexpected result: {:?}", other),
}
}
#[tokio::test]
async fn test_install_from_artifact_rejects_disallowed_host() {
let temp = tempfile::tempdir().expect("tempdir");
let installer = RegistryInstaller::new(
temp.path().to_path_buf(),
temp.path().join("tools"),
temp.path().join("channels"),
);
let manifest = test_manifest(
"demo",
"tools-src/demo",
Some("https://169.254.169.254/latest/meta-data".to_string()),
None,
);
let result = installer.install_from_artifact(&manifest, false).await;
match result {
Err(RegistryError::InvalidManifest { field, .. }) => {
assert_eq!(field, "artifacts.wasm32-wasip2.url");
}
other => panic!("unexpected result: {:?}", other),
}
}
#[tokio::test]
async fn test_install_from_artifact_rejects_null_sha256() {
let temp = tempfile::tempdir().expect("tempdir");
let installer = RegistryInstaller::new(
temp.path().to_path_buf(),
temp.path().join("tools"),
temp.path().join("channels"),
);
// Valid URL but no sha256 — should be rejected before any download attempt
let manifest = test_manifest(
"demo",
"tools-src/demo",
Some(
"https://github.com/nearai/ironclaw/releases/latest/download/demo-wasm32-wasip2.tar.gz".to_string(),
),
None, // sha256 = null
);
let result = installer.install_from_artifact(&manifest, false).await;
match result {
Err(RegistryError::InvalidManifest { field, reason, .. }) => {
assert_eq!(field, "artifacts.wasm32-wasip2.sha256");
assert!(reason.contains("required"), "reason: {}", reason);
}
other => panic!("unexpected result: {:?}", other),
}
}
#[test]
fn test_should_attempt_source_fallback_policy() {
let download = RegistryError::DownloadFailed {
url: "https://github.com/nearai/ironclaw/releases/latest/download/demo.wasm"
.to_string(),
reason: "http status 404".to_string(),
};
assert!(should_attempt_source_fallback(&download));
let already = RegistryError::AlreadyInstalled {
name: "demo".to_string(),
path: PathBuf::from("/tmp/demo.wasm"),
};
assert!(!should_attempt_source_fallback(&already));
let checksum = RegistryError::ChecksumMismatch {
url: "https://github.com/nearai/ironclaw/releases/latest/download/demo.wasm"
.to_string(),
expected_sha256: "deadbeef".to_string(),
actual_sha256: "feedface".to_string(),
};
assert!(!should_attempt_source_fallback(&checksum));
let invalid = RegistryError::InvalidManifest {
name: "demo".to_string(),
field: "artifacts.wasm32-wasip2.url",
reason: "host not allowed".to_string(),
};
assert!(!should_attempt_source_fallback(&invalid));
}
#[test]
@@ -587,6 +986,66 @@ mod tests {
assert!(result.has_capabilities);
}
#[tokio::test]
async fn test_install_from_source_rejects_wrong_prefix_for_channel() {
let temp = tempfile::tempdir().expect("tempdir");
let installer = RegistryInstaller::new(
temp.path().to_path_buf(),
temp.path().join("tools"),
temp.path().join("channels"),
);
// Channel manifest with tools-src/ prefix should be rejected
let manifest = test_manifest_with_kind(
"telegram",
"tools-src/telegram",
None,
None,
ManifestKind::Channel,
);
let result = installer.install_from_source(&manifest, false).await;
match result {
Err(RegistryError::InvalidManifest { field, reason, .. }) => {
assert_eq!(field, "source.dir");
assert!(reason.contains("channels-src/"), "reason: {}", reason);
}
other => panic!("unexpected result: {:?}", other),
}
}
#[tokio::test]
async fn test_install_from_source_accepts_correct_channel_prefix() {
let temp = tempfile::tempdir().expect("tempdir");
let installer = RegistryInstaller::new(
temp.path().to_path_buf(),
temp.path().join("tools"),
temp.path().join("channels"),
);
// Channel manifest with channels-src/ prefix should pass validation
// (will fail later because source dir doesn't exist, which is fine)
let manifest = test_manifest_with_kind(
"telegram",
"channels-src/telegram",
None,
None,
ManifestKind::Channel,
);
let result = installer.install_from_source(&manifest, false).await;
match result {
Err(RegistryError::ManifestRead { reason, .. }) => {
assert!(
reason.contains("source directory does not exist"),
"reason: {}",
reason
);
}
other => panic!("unexpected result: {:?}", other),
}
}
#[test]
fn test_extract_tar_gz_missing_wasm() {
use flate2::Compression;