mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-25 14:53:34 +00:00
feat(skills): compile-time skill bundling infrastructure
Add support for embedding skills into the binary at compile time: - build.rs: embed_skills() collects skills/*/SKILL.md into embedded_skills.json - src/skills/bundled.rs: loads embedded skills via include_str! - SkillRegistry: with_bundled_content(), load_from_content(), step 4 in discover_all() - Bundled skills are Trusted (ship with binary), lowest discovery priority - 4 new tests for bundled loading, user override, gating, and removal rejection - Cargo.toml: add serde_json build-dependency [skip-regression-check] Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
This commit is contained in:
@@ -195,6 +195,9 @@ security-framework = "3"
|
||||
secret-service = { version = "4", features = ["rt-tokio-crypto-rust"] }
|
||||
zbus = "4"
|
||||
|
||||
[build-dependencies]
|
||||
serde_json = "1"
|
||||
|
||||
[dev-dependencies]
|
||||
tokio-test = "0.4"
|
||||
tracing-test = "0.2"
|
||||
|
||||
@@ -20,6 +20,9 @@ fn main() {
|
||||
// ── Embed registry manifests ────────────────────────────────────────
|
||||
embed_registry_catalog(&root);
|
||||
|
||||
// ── Embed bundled skills ────────────────────────────────────────────
|
||||
embed_skills(&root);
|
||||
|
||||
// ── Build Telegram channel WASM ─────────────────────────────────────
|
||||
let channel_dir = root.join("channels-src/telegram");
|
||||
let wasm_out = channel_dir.join("telegram.wasm");
|
||||
@@ -125,7 +128,7 @@ fn embed_registry_catalog(root: &Path) {
|
||||
// are emitted inside collect_json_files to track content changes reliably).
|
||||
println!("cargo:rerun-if-changed=registry/_bundles.json");
|
||||
|
||||
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap());
|
||||
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); // safety: build script
|
||||
let out_path = out_dir.join("embedded_catalog.json");
|
||||
|
||||
if !registry_dir.is_dir() {
|
||||
@@ -177,7 +180,60 @@ fn embed_registry_catalog(root: &Path) {
|
||||
bundles_raw,
|
||||
);
|
||||
|
||||
fs::write(&out_path, catalog).unwrap();
|
||||
fs::write(&out_path, catalog).unwrap(); // safety: build script
|
||||
}
|
||||
|
||||
/// Collect all `skills/*/SKILL.md` files into an embedded JSON blob.
|
||||
///
|
||||
/// Output: `$OUT_DIR/embedded_skills.json` — a JSON array of `{"name": "...", "content": "..."}`.
|
||||
/// These are loaded at runtime as bundled skills (lowest discovery priority, Trusted trust level).
|
||||
fn embed_skills(root: &Path) {
|
||||
use std::fs;
|
||||
|
||||
let skills_dir = root.join("skills");
|
||||
|
||||
// Rerun when any skill changes
|
||||
println!("cargo:rerun-if-changed=skills");
|
||||
|
||||
let out_dir = PathBuf::from(env::var("OUT_DIR").unwrap()); // safety: build script panics on failure
|
||||
let out_path = out_dir.join("embedded_skills.json");
|
||||
|
||||
if !skills_dir.is_dir() {
|
||||
fs::write(&out_path, "[]").unwrap(); // safety: build script
|
||||
return;
|
||||
}
|
||||
|
||||
let mut skills: Vec<String> = Vec::new();
|
||||
|
||||
let mut entries: Vec<_> = fs::read_dir(&skills_dir)
|
||||
.unwrap() // safety: build script
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| e.path().is_dir())
|
||||
.collect();
|
||||
entries.sort_by_key(|e| e.file_name());
|
||||
|
||||
for entry in entries {
|
||||
let skill_md = entry.path().join("SKILL.md");
|
||||
if !skill_md.is_file() {
|
||||
continue;
|
||||
}
|
||||
// Emit per-file watch
|
||||
println!("cargo:rerun-if-changed={}", skill_md.display());
|
||||
|
||||
let name = entry.file_name().to_string_lossy().to_string();
|
||||
if let Ok(content) = fs::read_to_string(&skill_md) {
|
||||
// Escape for JSON embedding
|
||||
let name_json = serde_json::to_string(&name).unwrap(); // safety: build script
|
||||
let content_json = serde_json::to_string(&content).unwrap(); // safety: build script
|
||||
skills.push(format!(
|
||||
r#"{{"name":{},"content":{}}}"#,
|
||||
name_json, content_json
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let catalog = format!("[{}]", skills.join(","));
|
||||
fs::write(&out_path, catalog).unwrap(); // safety: build script
|
||||
}
|
||||
|
||||
/// Read all .json files from a directory and push their raw contents into `out`.
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
//! Skill registry for discovering, loading, and managing available skills.
|
||||
//!
|
||||
//! Skills are discovered from two filesystem locations:
|
||||
//! Skills are discovered from multiple sources:
|
||||
//! 1. Workspace skills directory (`<workspace>/skills/`) -- Trusted
|
||||
//! 2. User skills directory (`~/.ironclaw/skills/`) -- Trusted
|
||||
//! 3. Installed skills directory (`~/.ironclaw/installed_skills/`) -- Installed
|
||||
//! 4. Bundled skills compiled into the binary -- Trusted
|
||||
//!
|
||||
//! Both flat (`skills/SKILL.md`) and subdirectory (`skills/<name>/SKILL.md`)
|
||||
//! layouts are supported. Earlier locations win on name collision (workspace
|
||||
//! overrides user). Uses async I/O throughout to avoid blocking the tokio runtime.
|
||||
//! layouts are supported. Earlier sources win on name collision (workspace
|
||||
//! overrides user overrides installed overrides bundled).
|
||||
//! Uses async I/O throughout to avoid blocking the tokio runtime.
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -78,6 +81,9 @@ pub struct SkillRegistry {
|
||||
installed_dir: Option<PathBuf>,
|
||||
/// Optional workspace skills directory.
|
||||
workspace_dir: Option<PathBuf>,
|
||||
/// Bundled skill content compiled into the binary (name, raw SKILL.md content).
|
||||
/// Loaded as Trusted at lowest discovery priority.
|
||||
bundled_content: &'static [(String, String)],
|
||||
}
|
||||
|
||||
impl SkillRegistry {
|
||||
@@ -88,6 +94,7 @@ impl SkillRegistry {
|
||||
user_dir,
|
||||
installed_dir: None,
|
||||
workspace_dir: None,
|
||||
bundled_content: &[],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -108,6 +115,16 @@ impl SkillRegistry {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set bundled skill content compiled into the binary.
|
||||
///
|
||||
/// Each entry is `(skill_name, raw_skill_md_content)`. These skills are
|
||||
/// discovered at the lowest priority (after workspace, user, and installed)
|
||||
/// with `SkillTrust::Trusted` since they ship with the application binary.
|
||||
pub fn with_bundled_content(mut self, content: &'static [(String, String)]) -> Self {
|
||||
self.bundled_content = content;
|
||||
self
|
||||
}
|
||||
|
||||
/// Discover and load skills from all configured directories.
|
||||
///
|
||||
/// Discovery order (earlier wins on name collision):
|
||||
@@ -148,7 +165,7 @@ impl SkillRegistry {
|
||||
self.skills.push(skill);
|
||||
}
|
||||
|
||||
// 3. Installed skills (registry-installed, lowest priority)
|
||||
// 3. Installed skills (registry-installed)
|
||||
if let Some(inst_dir) = self.installed_dir.clone() {
|
||||
let inst_skills = self
|
||||
.discover_from_dir(&inst_dir, SkillTrust::Installed, SkillSource::User)
|
||||
@@ -167,6 +184,16 @@ impl SkillRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Bundled skills (compiled into binary, lowest priority)
|
||||
if !self.bundled_content.is_empty() {
|
||||
let bundled = self.load_bundled_skills(&seen).await;
|
||||
for (name, skill) in bundled {
|
||||
seen.insert(name.clone());
|
||||
loaded_names.push(name);
|
||||
self.skills.push(skill);
|
||||
}
|
||||
}
|
||||
|
||||
loaded_names
|
||||
}
|
||||
|
||||
@@ -282,6 +309,36 @@ impl SkillRegistry {
|
||||
load_and_validate_skill(path, trust, source).await
|
||||
}
|
||||
|
||||
/// Load bundled skills from in-memory content, skipping names already seen.
|
||||
async fn load_bundled_skills(&self, seen: &HashSet<String>) -> Vec<(String, LoadedSkill)> {
|
||||
let mut results = Vec::new();
|
||||
for (name, content) in self.bundled_content {
|
||||
if seen.contains(name) {
|
||||
tracing::debug!(
|
||||
"Skipping bundled skill '{}' (overridden by user/workspace/installed)",
|
||||
name
|
||||
);
|
||||
continue;
|
||||
}
|
||||
match load_from_content(
|
||||
content,
|
||||
SkillTrust::Trusted,
|
||||
SkillSource::Bundled(PathBuf::from(name)),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok((loaded_name, skill)) => {
|
||||
tracing::debug!("Loaded bundled skill: {}", loaded_name);
|
||||
results.push((loaded_name, skill));
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::debug!("Skipping bundled skill '{}': {}", name, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
results
|
||||
}
|
||||
|
||||
/// Get all loaded skills.
|
||||
pub fn skills(&self) -> &[LoadedSkill] {
|
||||
&self.skills
|
||||
@@ -606,6 +663,84 @@ async fn load_and_validate_skill(
|
||||
Ok((name, skill))
|
||||
}
|
||||
|
||||
/// Load and validate a skill from in-memory content (no disk I/O).
|
||||
///
|
||||
/// Used for bundled skills compiled into the binary.
|
||||
async fn load_from_content(
|
||||
raw_content: &str,
|
||||
trust: SkillTrust,
|
||||
source: SkillSource,
|
||||
) -> Result<(String, LoadedSkill), SkillRegistryError> {
|
||||
if raw_content.len() as u64 > MAX_PROMPT_FILE_SIZE {
|
||||
return Err(SkillRegistryError::FileTooLarge {
|
||||
name: "(bundled)".to_string(),
|
||||
size: raw_content.len() as u64,
|
||||
max: MAX_PROMPT_FILE_SIZE,
|
||||
});
|
||||
}
|
||||
|
||||
let normalized_content = normalize_line_endings(raw_content);
|
||||
|
||||
let parsed = parse_skill_md(&normalized_content).map_err(|e: SkillParseError| match e {
|
||||
SkillParseError::InvalidName { ref name } => SkillRegistryError::ParseError {
|
||||
name: name.clone(),
|
||||
reason: e.to_string(),
|
||||
},
|
||||
_ => SkillRegistryError::ParseError {
|
||||
name: "(bundled)".to_string(),
|
||||
reason: e.to_string(),
|
||||
},
|
||||
})?;
|
||||
|
||||
let manifest = parsed.manifest;
|
||||
let prompt_content = parsed.prompt_content;
|
||||
|
||||
// Check gating requirements
|
||||
if let Some(ref meta) = manifest.metadata
|
||||
&& let Some(ref openclaw) = meta.openclaw
|
||||
{
|
||||
let result = gating::check_requirements(&openclaw.requires).await;
|
||||
if !result.passed {
|
||||
return Err(SkillRegistryError::GatingFailed {
|
||||
name: manifest.name.clone(),
|
||||
reason: result.failures.join("; "),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Check token budget
|
||||
let approx_tokens = (prompt_content.len() as f64 * 0.25) as usize;
|
||||
let declared = manifest.activation.max_context_tokens;
|
||||
if declared > 0 && approx_tokens > declared * 2 {
|
||||
return Err(SkillRegistryError::TokenBudgetExceeded {
|
||||
name: manifest.name.clone(),
|
||||
approx_tokens,
|
||||
declared,
|
||||
});
|
||||
}
|
||||
|
||||
let content_hash = compute_hash(&prompt_content);
|
||||
let compiled_patterns = LoadedSkill::compile_patterns(&manifest.activation.patterns);
|
||||
let lowercased_keywords = to_lowercase_vec(&manifest.activation.keywords);
|
||||
let lowercased_exclude_keywords = to_lowercase_vec(&manifest.activation.exclude_keywords);
|
||||
let lowercased_tags = to_lowercase_vec(&manifest.activation.tags);
|
||||
|
||||
let name = manifest.name.clone();
|
||||
let skill = LoadedSkill {
|
||||
manifest,
|
||||
prompt_content,
|
||||
trust,
|
||||
source,
|
||||
content_hash,
|
||||
compiled_patterns,
|
||||
lowercased_keywords,
|
||||
lowercased_exclude_keywords,
|
||||
lowercased_tags,
|
||||
};
|
||||
|
||||
Ok((name, skill))
|
||||
}
|
||||
|
||||
/// Compute SHA-256 hash of content in the format "sha256:hex...".
|
||||
pub fn compute_hash(content: &str) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
@@ -616,9 +751,7 @@ pub fn compute_hash(content: &str) -> String {
|
||||
|
||||
/// Helper to check gating for a `GatingRequirements`. Useful for callers that
|
||||
/// don't have the full skill loaded yet.
|
||||
pub async fn check_gating(
|
||||
requirements: &GatingRequirements,
|
||||
) -> crate::gating::GatingResult {
|
||||
pub async fn check_gating(requirements: &GatingRequirements) -> crate::gating::GatingResult {
|
||||
gating::check_requirements(requirements).await
|
||||
}
|
||||
|
||||
@@ -1091,4 +1224,96 @@ mod tests {
|
||||
let skill = registry.find_by_name("my-skill").unwrap();
|
||||
assert_eq!(skill.trust, SkillTrust::Trusted);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bundled_skills_loaded() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
// Leak the vec so we get a &'static slice
|
||||
let bundled: &'static [(String, String)] = Box::leak(Box::new(vec![(
|
||||
"bundled-skill".to_string(),
|
||||
"---\nname: bundled-skill\ndescription: A bundled test\nactivation:\n keywords: [\"test\"]\n---\n\nBundled prompt.\n".to_string(),
|
||||
)]));
|
||||
|
||||
let mut registry =
|
||||
SkillRegistry::new(dir.path().to_path_buf()).with_bundled_content(bundled);
|
||||
let loaded = registry.discover_all().await;
|
||||
|
||||
assert_eq!(loaded, vec!["bundled-skill"]);
|
||||
assert_eq!(registry.count(), 1);
|
||||
|
||||
let skill = registry.find_by_name("bundled-skill").unwrap();
|
||||
assert_eq!(skill.trust, SkillTrust::Trusted);
|
||||
assert!(matches!(skill.source, SkillSource::Bundled(_)));
|
||||
assert!(skill.prompt_content.contains("Bundled prompt."));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bundled_skill_overridden_by_user() {
|
||||
let user_dir = tempfile::tempdir().unwrap();
|
||||
|
||||
// User skill
|
||||
let skill_dir = user_dir.path().join("my-skill");
|
||||
fs::create_dir(&skill_dir).unwrap();
|
||||
fs::write(
|
||||
skill_dir.join("SKILL.md"),
|
||||
"---\nname: my-skill\n---\n\nUser version.\n",
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Bundled skill with same name
|
||||
let bundled: &'static [(String, String)] = Box::leak(Box::new(vec![(
|
||||
"my-skill".to_string(),
|
||||
"---\nname: my-skill\n---\n\nBundled version.\n".to_string(),
|
||||
)]));
|
||||
|
||||
let mut registry =
|
||||
SkillRegistry::new(user_dir.path().to_path_buf()).with_bundled_content(bundled);
|
||||
let loaded = registry.discover_all().await;
|
||||
|
||||
assert_eq!(loaded, vec!["my-skill"]);
|
||||
assert_eq!(registry.count(), 1);
|
||||
// User version wins over bundled
|
||||
assert!(
|
||||
registry.skills()[0]
|
||||
.prompt_content
|
||||
.contains("User version.")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bundled_skill_gating_failure_skipped() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let bundled: &'static [(String, String)] = Box::leak(Box::new(vec![(
|
||||
"gated".to_string(),
|
||||
"---\nname: gated\nmetadata:\n openclaw:\n requires:\n bins: [\"__nonexistent__\"]\n---\n\nGated.\n".to_string(),
|
||||
)]));
|
||||
|
||||
let mut registry =
|
||||
SkillRegistry::new(dir.path().to_path_buf()).with_bundled_content(bundled);
|
||||
let loaded = registry.discover_all().await;
|
||||
|
||||
assert!(loaded.is_empty(), "gated bundled skill should be skipped");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_bundled_skill_cannot_be_removed() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let bundled: &'static [(String, String)] = Box::leak(Box::new(vec![(
|
||||
"permanent".to_string(),
|
||||
"---\nname: permanent\n---\n\nCannot remove.\n".to_string(),
|
||||
)]));
|
||||
|
||||
let mut registry =
|
||||
SkillRegistry::new(dir.path().to_path_buf()).with_bundled_content(bundled);
|
||||
registry.discover_all().await;
|
||||
|
||||
let result = registry.remove_skill("permanent").await;
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(SkillRegistryError::CannotRemove { .. })
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
+2
-1
@@ -864,7 +864,8 @@ impl AppBuilder {
|
||||
// Skills system
|
||||
let (skill_registry, skill_catalog) = if self.config.skills.enabled {
|
||||
let mut registry = SkillRegistry::new(self.config.skills.local_dir.clone())
|
||||
.with_installed_dir(self.config.skills.installed_dir.clone());
|
||||
.with_installed_dir(self.config.skills.installed_dir.clone())
|
||||
.with_bundled_content(crate::skills::bundled::load_bundled_skills());
|
||||
let loaded = registry.discover_all().await;
|
||||
if !loaded.is_empty() {
|
||||
tracing::debug!("Loaded {} skill(s): {}", loaded.len(), loaded.join(", "));
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
//! Bundled skills embedded into the binary at compile time.
|
||||
//!
|
||||
//! The `build.rs` script collects all `skills/*/SKILL.md` files into
|
||||
//! `embedded_skills.json`. This module deserializes that blob and provides
|
||||
//! the raw (name, content) pairs to the skill registry for discovery.
|
||||
|
||||
use std::sync::OnceLock;
|
||||
|
||||
/// Raw JSON generated by build.rs from `skills/*/SKILL.md`.
|
||||
const EMBEDDED_SKILLS_JSON: &str = include_str!(concat!(env!("OUT_DIR"), "/embedded_skills.json"));
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct EmbeddedSkillEntry {
|
||||
name: String,
|
||||
content: String,
|
||||
}
|
||||
|
||||
/// Parsed bundled skills cached across calls.
|
||||
fn parsed_skills() -> &'static Vec<(String, String)> {
|
||||
static CACHE: OnceLock<Vec<(String, String)>> = OnceLock::new();
|
||||
CACHE.get_or_init(|| {
|
||||
let entries: Vec<EmbeddedSkillEntry> = match serde_json::from_str(EMBEDDED_SKILLS_JSON) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to parse embedded skills catalog: {}", e);
|
||||
return Vec::new();
|
||||
}
|
||||
};
|
||||
entries.into_iter().map(|e| (e.name, e.content)).collect()
|
||||
})
|
||||
}
|
||||
|
||||
/// Load all bundled skill (name, content) pairs compiled into the binary.
|
||||
///
|
||||
/// Returns a slice of `(skill_name, skill_md_content)` tuples. These are
|
||||
/// loaded by the skill registry as the lowest-priority discovery source
|
||||
/// with `Trusted` trust level (they ship with the application).
|
||||
pub fn load_bundled_skills() -> &'static [(String, String)] {
|
||||
parsed_skills()
|
||||
}
|
||||
@@ -26,6 +26,7 @@
|
||||
//! The `ironclaw_skills` crate itself remains (types, parser, validation, v2 types).
|
||||
|
||||
pub mod attenuation;
|
||||
pub mod bundled;
|
||||
|
||||
// Re-export everything from the extracted crate.
|
||||
pub use ironclaw_skills::*;
|
||||
|
||||
Reference in New Issue
Block a user