feat(skills): compile-time skill bundling infrastructure

Add support for embedding skills into the binary at compile time:

- build.rs: embed_skills() collects skills/*/SKILL.md into embedded_skills.json
- src/skills/bundled.rs: loads embedded skills via include_str!
- SkillRegistry: with_bundled_content(), load_from_content(), step 4 in discover_all()
- Bundled skills are Trusted (ship with binary), lowest discovery priority
- 4 new tests for bundled loading, user override, gating, and removal rejection
- Cargo.toml: add serde_json build-dependency

[skip-regression-check]

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
This commit is contained in:
2026-03-27 20:14:46 -07:00
co-authored by Claude Opus 4.6
parent 49e83dec5e
commit 6eed4b722c
6 changed files with 336 additions and 10 deletions
+3
View File
@@ -195,6 +195,9 @@ security-framework = "3"
secret-service = { version = "4", features = ["rt-tokio-crypto-rust"] }
zbus = "4"
[build-dependencies]
serde_json = "1"
[dev-dependencies]
tokio-test = "0.4"
tracing-test = "0.2"
+58 -2
View File
@@ -20,6 +20,9 @@ fn main() {
// ── Embed registry manifests ────────────────────────────────────────
embed_registry_catalog(&root);
// ── Embed bundled skills ────────────────────────────────────────────
embed_skills(&root);
// ── Build Telegram channel WASM ─────────────────────────────────────
let channel_dir = root.join("channels-src/telegram");
let wasm_out = channel_dir.join("telegram.wasm");
@@ -125,7 +128,7 @@ fn embed_registry_catalog(root: &Path) {
// are emitted inside collect_json_files to track content changes reliably).
println!("cargo:rerun-if-changed=registry/_bundles.json");
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); // safety: build script
let out_path = out_dir.join("embedded_catalog.json");
if !registry_dir.is_dir() {
@@ -177,7 +180,60 @@ fn embed_registry_catalog(root: &Path) {
bundles_raw,
);
fs::write(&out_path, catalog).unwrap();
fs::write(&out_path, catalog).unwrap(); // safety: build script
}
/// Collect all `skills/*/SKILL.md` files into an embedded JSON blob.
///
/// Output: `$OUT_DIR/embedded_skills.json` — a JSON array of `{"name": "...", "content": "..."}`.
/// These are loaded at runtime as bundled skills (lowest discovery priority, Trusted trust level).
fn embed_skills(root: &Path) {
use std::fs;
let skills_dir = root.join("skills");
// Rerun when any skill changes
println!("cargo:rerun-if-changed=skills");
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); // safety: build script panics on failure
let out_path = out_dir.join("embedded_skills.json");
if !skills_dir.is_dir() {
fs::write(&out_path, "[]").unwrap(); // safety: build script
return;
}
let mut skills: Vec<String> = Vec::new();
let mut entries: Vec<_> = fs::read_dir(&skills_dir)
.unwrap() // safety: build script
.filter_map(|e| e.ok())
.filter(|e| e.path().is_dir())
.collect();
entries.sort_by_key(|e| e.file_name());
for entry in entries {
let skill_md = entry.path().join("SKILL.md");
if !skill_md.is_file() {
continue;
}
// Emit per-file watch
println!("cargo:rerun-if-changed={}", skill_md.display());
let name = entry.file_name().to_string_lossy().to_string();
if let Ok(content) = fs::read_to_string(&skill_md) {
// Escape for JSON embedding
let name_json = serde_json::to_string(&name).unwrap(); // safety: build script
let content_json = serde_json::to_string(&content).unwrap(); // safety: build script
skills.push(format!(
r#"{{"name":{},"content":{}}}"#,
name_json, content_json
));
}
}
let catalog = format!("[{}]", skills.join(","));
fs::write(&out_path, catalog).unwrap(); // safety: build script
}
/// Read all .json files from a directory and push their raw contents into `out`.
+232 -7
View File
@@ -1,12 +1,15 @@
//! Skill registry for discovering, loading, and managing available skills.
//!
//! Skills are discovered from two filesystem locations:
//! Skills are discovered from multiple sources:
//! 1. Workspace skills directory (`<workspace>/skills/`) -- Trusted
//! 2. User skills directory (`~/.ironclaw/skills/`) -- Trusted
//! 3. Installed skills directory (`~/.ironclaw/installed_skills/`) -- Installed
//! 4. Bundled skills compiled into the binary -- Trusted
//!
//! Both flat (`skills/SKILL.md`) and subdirectory (`skills/<name>/SKILL.md`)
//! layouts are supported. Earlier locations win on name collision (workspace
//! overrides user). Uses async I/O throughout to avoid blocking the tokio runtime.
//! layouts are supported. Earlier sources win on name collision (workspace
//! overrides user overrides installed overrides bundled).
//! Uses async I/O throughout to avoid blocking the tokio runtime.
use std::collections::HashSet;
use std::path::{Path, PathBuf};
@@ -78,6 +81,9 @@ pub struct SkillRegistry {
installed_dir: Option<PathBuf>,
/// Optional workspace skills directory.
workspace_dir: Option<PathBuf>,
/// Bundled skill content compiled into the binary (name, raw SKILL.md content).
/// Loaded as Trusted at lowest discovery priority.
bundled_content: &'static [(String, String)],
}
impl SkillRegistry {
@@ -88,6 +94,7 @@ impl SkillRegistry {
user_dir,
installed_dir: None,
workspace_dir: None,
bundled_content: &[],
}
}
@@ -108,6 +115,16 @@ impl SkillRegistry {
self
}
/// Set bundled skill content compiled into the binary.
///
/// Each entry is `(skill_name, raw_skill_md_content)`. These skills are
/// discovered at the lowest priority (after workspace, user, and installed)
/// with `SkillTrust::Trusted` since they ship with the application binary.
pub fn with_bundled_content(mut self, content: &'static [(String, String)]) -> Self {
self.bundled_content = content;
self
}
/// Discover and load skills from all configured directories.
///
/// Discovery order (earlier wins on name collision):
@@ -148,7 +165,7 @@ impl SkillRegistry {
self.skills.push(skill);
}
// 3. Installed skills (registry-installed, lowest priority)
// 3. Installed skills (registry-installed)
if let Some(inst_dir) = self.installed_dir.clone() {
let inst_skills = self
.discover_from_dir(&inst_dir, SkillTrust::Installed, SkillSource::User)
@@ -167,6 +184,16 @@ impl SkillRegistry {
}
}
// 4. Bundled skills (compiled into binary, lowest priority)
if !self.bundled_content.is_empty() {
let bundled = self.load_bundled_skills(&seen).await;
for (name, skill) in bundled {
seen.insert(name.clone());
loaded_names.push(name);
self.skills.push(skill);
}
}
loaded_names
}
@@ -282,6 +309,36 @@ impl SkillRegistry {
load_and_validate_skill(path, trust, source).await
}
/// Load bundled skills from in-memory content, skipping names already seen.
async fn load_bundled_skills(&self, seen: &HashSet<String>) -> Vec<(String, LoadedSkill)> {
let mut results = Vec::new();
for (name, content) in self.bundled_content {
if seen.contains(name) {
tracing::debug!(
"Skipping bundled skill '{}' (overridden by user/workspace/installed)",
name
);
continue;
}
match load_from_content(
content,
SkillTrust::Trusted,
SkillSource::Bundled(PathBuf::from(name)),
)
.await
{
Ok((loaded_name, skill)) => {
tracing::debug!("Loaded bundled skill: {}", loaded_name);
results.push((loaded_name, skill));
}
Err(e) => {
tracing::debug!("Skipping bundled skill '{}': {}", name, e);
}
}
}
results
}
/// Get all loaded skills.
pub fn skills(&self) -> &[LoadedSkill] {
&self.skills
@@ -606,6 +663,84 @@ async fn load_and_validate_skill(
Ok((name, skill))
}
/// Load and validate a skill from in-memory content (no disk I/O).
///
/// Used for bundled skills compiled into the binary.
async fn load_from_content(
raw_content: &str,
trust: SkillTrust,
source: SkillSource,
) -> Result<(String, LoadedSkill), SkillRegistryError> {
if raw_content.len() as u64 > MAX_PROMPT_FILE_SIZE {
return Err(SkillRegistryError::FileTooLarge {
name: "(bundled)".to_string(),
size: raw_content.len() as u64,
max: MAX_PROMPT_FILE_SIZE,
});
}
let normalized_content = normalize_line_endings(raw_content);
let parsed = parse_skill_md(&normalized_content).map_err(|e: SkillParseError| match e {
SkillParseError::InvalidName { ref name } => SkillRegistryError::ParseError {
name: name.clone(),
reason: e.to_string(),
},
_ => SkillRegistryError::ParseError {
name: "(bundled)".to_string(),
reason: e.to_string(),
},
})?;
let manifest = parsed.manifest;
let prompt_content = parsed.prompt_content;
// Check gating requirements
if let Some(ref meta) = manifest.metadata
&& let Some(ref openclaw) = meta.openclaw
{
let result = gating::check_requirements(&openclaw.requires).await;
if !result.passed {
return Err(SkillRegistryError::GatingFailed {
name: manifest.name.clone(),
reason: result.failures.join("; "),
});
}
}
// Check token budget
let approx_tokens = (prompt_content.len() as f64 * 0.25) as usize;
let declared = manifest.activation.max_context_tokens;
if declared > 0 && approx_tokens > declared * 2 {
return Err(SkillRegistryError::TokenBudgetExceeded {
name: manifest.name.clone(),
approx_tokens,
declared,
});
}
let content_hash = compute_hash(&prompt_content);
let compiled_patterns = LoadedSkill::compile_patterns(&manifest.activation.patterns);
let lowercased_keywords = to_lowercase_vec(&manifest.activation.keywords);
let lowercased_exclude_keywords = to_lowercase_vec(&manifest.activation.exclude_keywords);
let lowercased_tags = to_lowercase_vec(&manifest.activation.tags);
let name = manifest.name.clone();
let skill = LoadedSkill {
manifest,
prompt_content,
trust,
source,
content_hash,
compiled_patterns,
lowercased_keywords,
lowercased_exclude_keywords,
lowercased_tags,
};
Ok((name, skill))
}
/// Compute SHA-256 hash of content in the format "sha256:hex...".
pub fn compute_hash(content: &str) -> String {
let mut hasher = Sha256::new();
@@ -616,9 +751,7 @@ pub fn compute_hash(content: &str) -> String {
/// Helper to check gating for a `GatingRequirements`. Useful for callers that
/// don't have the full skill loaded yet.
pub async fn check_gating(
requirements: &GatingRequirements,
) -> crate::gating::GatingResult {
pub async fn check_gating(requirements: &GatingRequirements) -> crate::gating::GatingResult {
gating::check_requirements(requirements).await
}
@@ -1091,4 +1224,96 @@ mod tests {
let skill = registry.find_by_name("my-skill").unwrap();
assert_eq!(skill.trust, SkillTrust::Trusted);
}
#[tokio::test]
async fn test_bundled_skills_loaded() {
let dir = tempfile::tempdir().unwrap();
// Leak the vec so we get a &'static slice
let bundled: &'static [(String, String)] = Box::leak(Box::new(vec![(
"bundled-skill".to_string(),
"---\nname: bundled-skill\ndescription: A bundled test\nactivation:\n keywords: [\"test\"]\n---\n\nBundled prompt.\n".to_string(),
)]));
let mut registry =
SkillRegistry::new(dir.path().to_path_buf()).with_bundled_content(bundled);
let loaded = registry.discover_all().await;
assert_eq!(loaded, vec!["bundled-skill"]);
assert_eq!(registry.count(), 1);
let skill = registry.find_by_name("bundled-skill").unwrap();
assert_eq!(skill.trust, SkillTrust::Trusted);
assert!(matches!(skill.source, SkillSource::Bundled(_)));
assert!(skill.prompt_content.contains("Bundled prompt."));
}
#[tokio::test]
async fn test_bundled_skill_overridden_by_user() {
let user_dir = tempfile::tempdir().unwrap();
// User skill
let skill_dir = user_dir.path().join("my-skill");
fs::create_dir(&skill_dir).unwrap();
fs::write(
skill_dir.join("SKILL.md"),
"---\nname: my-skill\n---\n\nUser version.\n",
)
.unwrap();
// Bundled skill with same name
let bundled: &'static [(String, String)] = Box::leak(Box::new(vec![(
"my-skill".to_string(),
"---\nname: my-skill\n---\n\nBundled version.\n".to_string(),
)]));
let mut registry =
SkillRegistry::new(user_dir.path().to_path_buf()).with_bundled_content(bundled);
let loaded = registry.discover_all().await;
assert_eq!(loaded, vec!["my-skill"]);
assert_eq!(registry.count(), 1);
// User version wins over bundled
assert!(
registry.skills()[0]
.prompt_content
.contains("User version.")
);
}
#[tokio::test]
async fn test_bundled_skill_gating_failure_skipped() {
let dir = tempfile::tempdir().unwrap();
let bundled: &'static [(String, String)] = Box::leak(Box::new(vec![(
"gated".to_string(),
"---\nname: gated\nmetadata:\n openclaw:\n requires:\n bins: [\"__nonexistent__\"]\n---\n\nGated.\n".to_string(),
)]));
let mut registry =
SkillRegistry::new(dir.path().to_path_buf()).with_bundled_content(bundled);
let loaded = registry.discover_all().await;
assert!(loaded.is_empty(), "gated bundled skill should be skipped");
}
#[tokio::test]
async fn test_bundled_skill_cannot_be_removed() {
let dir = tempfile::tempdir().unwrap();
let bundled: &'static [(String, String)] = Box::leak(Box::new(vec![(
"permanent".to_string(),
"---\nname: permanent\n---\n\nCannot remove.\n".to_string(),
)]));
let mut registry =
SkillRegistry::new(dir.path().to_path_buf()).with_bundled_content(bundled);
registry.discover_all().await;
let result = registry.remove_skill("permanent").await;
assert!(matches!(
result,
Err(SkillRegistryError::CannotRemove { .. })
));
}
}
+2 -1
View File
@@ -864,7 +864,8 @@ impl AppBuilder {
// Skills system
let (skill_registry, skill_catalog) = if self.config.skills.enabled {
let mut registry = SkillRegistry::new(self.config.skills.local_dir.clone())
.with_installed_dir(self.config.skills.installed_dir.clone());
.with_installed_dir(self.config.skills.installed_dir.clone())
.with_bundled_content(crate::skills::bundled::load_bundled_skills());
let loaded = registry.discover_all().await;
if !loaded.is_empty() {
tracing::debug!("Loaded {} skill(s): {}", loaded.len(), loaded.join(", "));
+40
View File
@@ -0,0 +1,40 @@
//! Bundled skills embedded into the binary at compile time.
//!
//! The `build.rs` script collects all `skills/*/SKILL.md` files into
//! `embedded_skills.json`. This module deserializes that blob and provides
//! the raw (name, content) pairs to the skill registry for discovery.
use std::sync::OnceLock;
/// Raw JSON generated by build.rs from `skills/*/SKILL.md`.
const EMBEDDED_SKILLS_JSON: &str = include_str!(concat!(env!("OUT_DIR"), "/embedded_skills.json"));
#[derive(serde::Deserialize)]
struct EmbeddedSkillEntry {
name: String,
content: String,
}
/// Parsed bundled skills cached across calls.
fn parsed_skills() -> &'static Vec<(String, String)> {
static CACHE: OnceLock<Vec<(String, String)>> = OnceLock::new();
CACHE.get_or_init(|| {
let entries: Vec<EmbeddedSkillEntry> = match serde_json::from_str(EMBEDDED_SKILLS_JSON) {
Ok(v) => v,
Err(e) => {
tracing::warn!("Failed to parse embedded skills catalog: {}", e);
return Vec::new();
}
};
entries.into_iter().map(|e| (e.name, e.content)).collect()
})
}
/// Load all bundled skill (name, content) pairs compiled into the binary.
///
/// Returns a slice of `(skill_name, skill_md_content)` tuples. These are
/// loaded by the skill registry as the lowest-priority discovery source
/// with `Trusted` trust level (they ship with the application).
pub fn load_bundled_skills() -> &'static [(String, String)] {
parsed_skills()
}
+1
View File
@@ -26,6 +26,7 @@
//! The `ironclaw_skills` crate itself remains (types, parser, validation, v2 types).
pub mod attenuation;
pub mod bundled;
// Re-export everything from the extracted crate.
pub use ironclaw_skills::*;