From 6eed4b722c996fb6c6305b1ce99919e3a536348d Mon Sep 17 00:00:00 2001 From: "ilblackdragon@gmail.com" Date: Fri, 27 Mar 2026 20:14:46 -0700 Subject: [PATCH] feat(skills): compile-time skill bundling infrastructure Add support for embedding skills into the binary at compile time: - build.rs: embed_skills() collects skills/*/SKILL.md into embedded_skills.json - src/skills/bundled.rs: loads embedded skills via include_str! - SkillRegistry: with_bundled_content(), load_from_content(), step 4 in discover_all() - Bundled skills are Trusted (ship with binary), lowest discovery priority - 4 new tests for bundled loading, user override, gating, and removal rejection - Cargo.toml: add serde_json build-dependency [skip-regression-check] Co-Authored-By: Claude Opus 4.6 (1M context) --- Cargo.toml | 3 + build.rs | 60 ++++++- crates/ironclaw_skills/src/registry.rs | 239 ++++++++++++++++++++++++- src/app.rs | 3 +- src/skills/bundled.rs | 40 +++++ src/skills/mod.rs | 1 + 6 files changed, 336 insertions(+), 10 deletions(-) create mode 100644 src/skills/bundled.rs diff --git a/Cargo.toml b/Cargo.toml index 95c3fd0b..ba8db332 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -195,6 +195,9 @@ security-framework = "3" secret-service = { version = "4", features = ["rt-tokio-crypto-rust"] } zbus = "4" +[build-dependencies] +serde_json = "1" + [dev-dependencies] tokio-test = "0.4" tracing-test = "0.2" diff --git a/build.rs b/build.rs index c2b93923..dbe50aaf 100644 --- a/build.rs +++ b/build.rs @@ -20,6 +20,9 @@ fn main() { // ── Embed registry manifests ──────────────────────────────────────── embed_registry_catalog(&root); + // ── Embed bundled skills ──────────────────────────────────────────── + embed_skills(&root); + // ── Build Telegram channel WASM ───────────────────────────────────── let channel_dir = root.join("channels-src/telegram"); let wasm_out = channel_dir.join("telegram.wasm"); @@ -125,7 +128,7 @@ fn embed_registry_catalog(root: &Path) { // are emitted inside collect_json_files to track content changes reliably). println!("cargo:rerun-if-changed=registry/_bundles.json"); - let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); // safety: build script let out_path = out_dir.join("embedded_catalog.json"); if !registry_dir.is_dir() { @@ -177,7 +180,60 @@ fn embed_registry_catalog(root: &Path) { bundles_raw, ); - fs::write(&out_path, catalog).unwrap(); + fs::write(&out_path, catalog).unwrap(); // safety: build script +} + +/// Collect all `skills/*/SKILL.md` files into an embedded JSON blob. +/// +/// Output: `$OUT_DIR/embedded_skills.json` — a JSON array of `{"name": "...", "content": "..."}`. +/// These are loaded at runtime as bundled skills (lowest discovery priority, Trusted trust level). +fn embed_skills(root: &Path) { + use std::fs; + + let skills_dir = root.join("skills"); + + // Rerun when any skill changes + println!("cargo:rerun-if-changed=skills"); + + let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); // safety: build script panics on failure + let out_path = out_dir.join("embedded_skills.json"); + + if !skills_dir.is_dir() { + fs::write(&out_path, "[]").unwrap(); // safety: build script + return; + } + + let mut skills: Vec = Vec::new(); + + let mut entries: Vec<_> = fs::read_dir(&skills_dir) + .unwrap() // safety: build script + .filter_map(|e| e.ok()) + .filter(|e| e.path().is_dir()) + .collect(); + entries.sort_by_key(|e| e.file_name()); + + for entry in entries { + let skill_md = entry.path().join("SKILL.md"); + if !skill_md.is_file() { + continue; + } + // Emit per-file watch + println!("cargo:rerun-if-changed={}", skill_md.display()); + + let name = entry.file_name().to_string_lossy().to_string(); + if let Ok(content) = fs::read_to_string(&skill_md) { + // Escape for JSON embedding + let name_json = serde_json::to_string(&name).unwrap(); // safety: build script + let content_json = serde_json::to_string(&content).unwrap(); // safety: build script + skills.push(format!( + r#"{{"name":{},"content":{}}}"#, + name_json, content_json + )); + } + } + + let catalog = format!("[{}]", skills.join(",")); + fs::write(&out_path, catalog).unwrap(); // safety: build script } /// Read all .json files from a directory and push their raw contents into `out`. diff --git a/crates/ironclaw_skills/src/registry.rs b/crates/ironclaw_skills/src/registry.rs index 51b9dfc9..16c3b2f5 100644 --- a/crates/ironclaw_skills/src/registry.rs +++ b/crates/ironclaw_skills/src/registry.rs @@ -1,12 +1,15 @@ //! Skill registry for discovering, loading, and managing available skills. //! -//! Skills are discovered from two filesystem locations: +//! Skills are discovered from multiple sources: //! 1. Workspace skills directory (`/skills/`) -- Trusted //! 2. User skills directory (`~/.ironclaw/skills/`) -- Trusted +//! 3. Installed skills directory (`~/.ironclaw/installed_skills/`) -- Installed +//! 4. Bundled skills compiled into the binary -- Trusted //! //! Both flat (`skills/SKILL.md`) and subdirectory (`skills//SKILL.md`) -//! layouts are supported. Earlier locations win on name collision (workspace -//! overrides user). Uses async I/O throughout to avoid blocking the tokio runtime. +//! layouts are supported. Earlier sources win on name collision (workspace +//! overrides user overrides installed overrides bundled). +//! Uses async I/O throughout to avoid blocking the tokio runtime. use std::collections::HashSet; use std::path::{Path, PathBuf}; @@ -78,6 +81,9 @@ pub struct SkillRegistry { installed_dir: Option, /// Optional workspace skills directory. workspace_dir: Option, + /// Bundled skill content compiled into the binary (name, raw SKILL.md content). + /// Loaded as Trusted at lowest discovery priority. + bundled_content: &'static [(String, String)], } impl SkillRegistry { @@ -88,6 +94,7 @@ impl SkillRegistry { user_dir, installed_dir: None, workspace_dir: None, + bundled_content: &[], } } @@ -108,6 +115,16 @@ impl SkillRegistry { self } + /// Set bundled skill content compiled into the binary. + /// + /// Each entry is `(skill_name, raw_skill_md_content)`. These skills are + /// discovered at the lowest priority (after workspace, user, and installed) + /// with `SkillTrust::Trusted` since they ship with the application binary. + pub fn with_bundled_content(mut self, content: &'static [(String, String)]) -> Self { + self.bundled_content = content; + self + } + /// Discover and load skills from all configured directories. /// /// Discovery order (earlier wins on name collision): @@ -148,7 +165,7 @@ impl SkillRegistry { self.skills.push(skill); } - // 3. Installed skills (registry-installed, lowest priority) + // 3. Installed skills (registry-installed) if let Some(inst_dir) = self.installed_dir.clone() { let inst_skills = self .discover_from_dir(&inst_dir, SkillTrust::Installed, SkillSource::User) @@ -167,6 +184,16 @@ impl SkillRegistry { } } + // 4. Bundled skills (compiled into binary, lowest priority) + if !self.bundled_content.is_empty() { + let bundled = self.load_bundled_skills(&seen).await; + for (name, skill) in bundled { + seen.insert(name.clone()); + loaded_names.push(name); + self.skills.push(skill); + } + } + loaded_names } @@ -282,6 +309,36 @@ impl SkillRegistry { load_and_validate_skill(path, trust, source).await } + /// Load bundled skills from in-memory content, skipping names already seen. + async fn load_bundled_skills(&self, seen: &HashSet) -> Vec<(String, LoadedSkill)> { + let mut results = Vec::new(); + for (name, content) in self.bundled_content { + if seen.contains(name) { + tracing::debug!( + "Skipping bundled skill '{}' (overridden by user/workspace/installed)", + name + ); + continue; + } + match load_from_content( + content, + SkillTrust::Trusted, + SkillSource::Bundled(PathBuf::from(name)), + ) + .await + { + Ok((loaded_name, skill)) => { + tracing::debug!("Loaded bundled skill: {}", loaded_name); + results.push((loaded_name, skill)); + } + Err(e) => { + tracing::debug!("Skipping bundled skill '{}': {}", name, e); + } + } + } + results + } + /// Get all loaded skills. pub fn skills(&self) -> &[LoadedSkill] { &self.skills @@ -606,6 +663,84 @@ async fn load_and_validate_skill( Ok((name, skill)) } +/// Load and validate a skill from in-memory content (no disk I/O). +/// +/// Used for bundled skills compiled into the binary. +async fn load_from_content( + raw_content: &str, + trust: SkillTrust, + source: SkillSource, +) -> Result<(String, LoadedSkill), SkillRegistryError> { + if raw_content.len() as u64 > MAX_PROMPT_FILE_SIZE { + return Err(SkillRegistryError::FileTooLarge { + name: "(bundled)".to_string(), + size: raw_content.len() as u64, + max: MAX_PROMPT_FILE_SIZE, + }); + } + + let normalized_content = normalize_line_endings(raw_content); + + let parsed = parse_skill_md(&normalized_content).map_err(|e: SkillParseError| match e { + SkillParseError::InvalidName { ref name } => SkillRegistryError::ParseError { + name: name.clone(), + reason: e.to_string(), + }, + _ => SkillRegistryError::ParseError { + name: "(bundled)".to_string(), + reason: e.to_string(), + }, + })?; + + let manifest = parsed.manifest; + let prompt_content = parsed.prompt_content; + + // Check gating requirements + if let Some(ref meta) = manifest.metadata + && let Some(ref openclaw) = meta.openclaw + { + let result = gating::check_requirements(&openclaw.requires).await; + if !result.passed { + return Err(SkillRegistryError::GatingFailed { + name: manifest.name.clone(), + reason: result.failures.join("; "), + }); + } + } + + // Check token budget + let approx_tokens = (prompt_content.len() as f64 * 0.25) as usize; + let declared = manifest.activation.max_context_tokens; + if declared > 0 && approx_tokens > declared * 2 { + return Err(SkillRegistryError::TokenBudgetExceeded { + name: manifest.name.clone(), + approx_tokens, + declared, + }); + } + + let content_hash = compute_hash(&prompt_content); + let compiled_patterns = LoadedSkill::compile_patterns(&manifest.activation.patterns); + let lowercased_keywords = to_lowercase_vec(&manifest.activation.keywords); + let lowercased_exclude_keywords = to_lowercase_vec(&manifest.activation.exclude_keywords); + let lowercased_tags = to_lowercase_vec(&manifest.activation.tags); + + let name = manifest.name.clone(); + let skill = LoadedSkill { + manifest, + prompt_content, + trust, + source, + content_hash, + compiled_patterns, + lowercased_keywords, + lowercased_exclude_keywords, + lowercased_tags, + }; + + Ok((name, skill)) +} + /// Compute SHA-256 hash of content in the format "sha256:hex...". pub fn compute_hash(content: &str) -> String { let mut hasher = Sha256::new(); @@ -616,9 +751,7 @@ pub fn compute_hash(content: &str) -> String { /// Helper to check gating for a `GatingRequirements`. Useful for callers that /// don't have the full skill loaded yet. -pub async fn check_gating( - requirements: &GatingRequirements, -) -> crate::gating::GatingResult { +pub async fn check_gating(requirements: &GatingRequirements) -> crate::gating::GatingResult { gating::check_requirements(requirements).await } @@ -1091,4 +1224,96 @@ mod tests { let skill = registry.find_by_name("my-skill").unwrap(); assert_eq!(skill.trust, SkillTrust::Trusted); } + + #[tokio::test] + async fn test_bundled_skills_loaded() { + let dir = tempfile::tempdir().unwrap(); + + // Leak the vec so we get a &'static slice + let bundled: &'static [(String, String)] = Box::leak(Box::new(vec![( + "bundled-skill".to_string(), + "---\nname: bundled-skill\ndescription: A bundled test\nactivation:\n keywords: [\"test\"]\n---\n\nBundled prompt.\n".to_string(), + )])); + + let mut registry = + SkillRegistry::new(dir.path().to_path_buf()).with_bundled_content(bundled); + let loaded = registry.discover_all().await; + + assert_eq!(loaded, vec!["bundled-skill"]); + assert_eq!(registry.count(), 1); + + let skill = registry.find_by_name("bundled-skill").unwrap(); + assert_eq!(skill.trust, SkillTrust::Trusted); + assert!(matches!(skill.source, SkillSource::Bundled(_))); + assert!(skill.prompt_content.contains("Bundled prompt.")); + } + + #[tokio::test] + async fn test_bundled_skill_overridden_by_user() { + let user_dir = tempfile::tempdir().unwrap(); + + // User skill + let skill_dir = user_dir.path().join("my-skill"); + fs::create_dir(&skill_dir).unwrap(); + fs::write( + skill_dir.join("SKILL.md"), + "---\nname: my-skill\n---\n\nUser version.\n", + ) + .unwrap(); + + // Bundled skill with same name + let bundled: &'static [(String, String)] = Box::leak(Box::new(vec![( + "my-skill".to_string(), + "---\nname: my-skill\n---\n\nBundled version.\n".to_string(), + )])); + + let mut registry = + SkillRegistry::new(user_dir.path().to_path_buf()).with_bundled_content(bundled); + let loaded = registry.discover_all().await; + + assert_eq!(loaded, vec!["my-skill"]); + assert_eq!(registry.count(), 1); + // User version wins over bundled + assert!( + registry.skills()[0] + .prompt_content + .contains("User version.") + ); + } + + #[tokio::test] + async fn test_bundled_skill_gating_failure_skipped() { + let dir = tempfile::tempdir().unwrap(); + + let bundled: &'static [(String, String)] = Box::leak(Box::new(vec![( + "gated".to_string(), + "---\nname: gated\nmetadata:\n openclaw:\n requires:\n bins: [\"__nonexistent__\"]\n---\n\nGated.\n".to_string(), + )])); + + let mut registry = + SkillRegistry::new(dir.path().to_path_buf()).with_bundled_content(bundled); + let loaded = registry.discover_all().await; + + assert!(loaded.is_empty(), "gated bundled skill should be skipped"); + } + + #[tokio::test] + async fn test_bundled_skill_cannot_be_removed() { + let dir = tempfile::tempdir().unwrap(); + + let bundled: &'static [(String, String)] = Box::leak(Box::new(vec![( + "permanent".to_string(), + "---\nname: permanent\n---\n\nCannot remove.\n".to_string(), + )])); + + let mut registry = + SkillRegistry::new(dir.path().to_path_buf()).with_bundled_content(bundled); + registry.discover_all().await; + + let result = registry.remove_skill("permanent").await; + assert!(matches!( + result, + Err(SkillRegistryError::CannotRemove { .. }) + )); + } } diff --git a/src/app.rs b/src/app.rs index 28937a54..4cc5da28 100644 --- a/src/app.rs +++ b/src/app.rs @@ -864,7 +864,8 @@ impl AppBuilder { // Skills system let (skill_registry, skill_catalog) = if self.config.skills.enabled { let mut registry = SkillRegistry::new(self.config.skills.local_dir.clone()) - .with_installed_dir(self.config.skills.installed_dir.clone()); + .with_installed_dir(self.config.skills.installed_dir.clone()) + .with_bundled_content(crate::skills::bundled::load_bundled_skills()); let loaded = registry.discover_all().await; if !loaded.is_empty() { tracing::debug!("Loaded {} skill(s): {}", loaded.len(), loaded.join(", ")); diff --git a/src/skills/bundled.rs b/src/skills/bundled.rs new file mode 100644 index 00000000..32429507 --- /dev/null +++ b/src/skills/bundled.rs @@ -0,0 +1,40 @@ +//! Bundled skills embedded into the binary at compile time. +//! +//! The `build.rs` script collects all `skills/*/SKILL.md` files into +//! `embedded_skills.json`. This module deserializes that blob and provides +//! the raw (name, content) pairs to the skill registry for discovery. + +use std::sync::OnceLock; + +/// Raw JSON generated by build.rs from `skills/*/SKILL.md`. +const EMBEDDED_SKILLS_JSON: &str = include_str!(concat!(env!("OUT_DIR"), "/embedded_skills.json")); + +#[derive(serde::Deserialize)] +struct EmbeddedSkillEntry { + name: String, + content: String, +} + +/// Parsed bundled skills cached across calls. +fn parsed_skills() -> &'static Vec<(String, String)> { + static CACHE: OnceLock> = OnceLock::new(); + CACHE.get_or_init(|| { + let entries: Vec = match serde_json::from_str(EMBEDDED_SKILLS_JSON) { + Ok(v) => v, + Err(e) => { + tracing::warn!("Failed to parse embedded skills catalog: {}", e); + return Vec::new(); + } + }; + entries.into_iter().map(|e| (e.name, e.content)).collect() + }) +} + +/// Load all bundled skill (name, content) pairs compiled into the binary. +/// +/// Returns a slice of `(skill_name, skill_md_content)` tuples. These are +/// loaded by the skill registry as the lowest-priority discovery source +/// with `Trusted` trust level (they ship with the application). +pub fn load_bundled_skills() -> &'static [(String, String)] { + parsed_skills() +} diff --git a/src/skills/mod.rs b/src/skills/mod.rs index 117a5dfa..931c822e 100644 --- a/src/skills/mod.rs +++ b/src/skills/mod.rs @@ -26,6 +26,7 @@ //! The `ironclaw_skills` crate itself remains (types, parser, validation, v2 types). pub mod attenuation; +pub mod bundled; // Re-export everything from the extracted crate. pub use ironclaw_skills::*;