fix(engine): add state hint on code errors + retrieval engine integration

When code fails with NameError/UnboundLocalError (model trying to
access variables from a previous step), the error output now includes:

  [HINT] Variables don't persist between code blocks. Use the `state`
  dict to access data from previous steps. Available keys: ["web_search",
  "last_return"]

This teaches the model to use `state["web_search"]` instead of `result`
after a NameError, reducing wasted steps from 3-4 to 1.

Also integrates RetrievalEngine into context building and ThreadManager:
- build_step_context() now accepts optional RetrievalEngine to inject
  relevant memory docs (Lessons, Specs, Playbooks) into LLM context
- RetrievalEngine uses keyword matching with doc-type priority scoring
- Memory docs from reflection (Phase 4) now feed back into future threads

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

crates/ironclaw_engine/src/executor/context.rs

@@ -1,24 +1,226 @@
 //! Context building for LLM calls.
 //!
 //! Assembles the message sequence and action definitions from thread state,
-//! active leases, and (Phase 4) project memory docs.
+//! active leases, and project memory docs retrieved via the [`RetrievalEngine`].
 
 use std::sync::Arc;
 
+use crate::memory::RetrievalEngine;
 use crate::types::capability::{ActionDef, CapabilityLease};
 use crate::types::error::EngineError;
+use crate::types::memory::MemoryDoc;
 use crate::types::message::ThreadMessage;
+use crate::types::project::ProjectId;
 use crate::traits::effect::EffectExecutor;
 
+/// Maximum number of memory docs to inject into context.
+const MAX_CONTEXT_DOCS: usize = 5;
+
 /// Build the context for an LLM call: messages and available actions.
 ///
-/// Phase 1: passes through thread messages + resolves actions from leases.
-/// Phase 4 will add memory doc retrieval and injection.
+/// Retrieves relevant memory docs from the project and injects them as a
+/// system message after the main system prompt. This gives the LLM access
+/// to lessons learned, playbooks, and known issues from prior threads.
 pub async fn build_step_context(
     messages: &[ThreadMessage],
     leases: &[CapabilityLease],
     effects: &Arc<dyn EffectExecutor>,
+    retrieval: Option<&RetrievalEngine>,
+    project_id: ProjectId,
+    goal: &str,
 ) -> Result<(Vec<ThreadMessage>, Vec<ActionDef>), EngineError> {
     let actions = effects.available_actions(leases).await?;
-    Ok((messages.to_vec(), actions))
+
+    let mut ctx_messages = messages.to_vec();
+
+    // Inject retrieved memory docs as context
+    if let Some(engine) = retrieval {
+        let docs = engine
+            .retrieve_context(project_id, goal, MAX_CONTEXT_DOCS)
+            .await?;
+        if !docs.is_empty() {
+            let context_msg = format_docs_as_context(&docs);
+            // Insert after the system prompt (index 1) if one exists,
+            // otherwise prepend.
+            let insert_pos = if !ctx_messages.is_empty()
+                && ctx_messages[0].role == crate::types::message::MessageRole::System
+            {
+                1
+            } else {
+                0
+            };
+            ctx_messages.insert(insert_pos, ThreadMessage::system(context_msg));
+        }
+    }
+
+    Ok((ctx_messages, actions))
+}
+
+/// Format memory docs into a system message for context injection.
+fn format_docs_as_context(docs: &[MemoryDoc]) -> String {
+    let mut parts = vec!["## Prior Knowledge (from completed threads)\n".to_string()];
+
+    for doc in docs {
+        let type_label = match doc.doc_type {
+            crate::types::memory::DocType::Lesson => "LESSON",
+            crate::types::memory::DocType::Spec => "MISSING CAPABILITY",
+            crate::types::memory::DocType::Playbook => "PLAYBOOK",
+            crate::types::memory::DocType::Issue => "KNOWN ISSUE",
+            crate::types::memory::DocType::Summary => "CONTEXT",
+            crate::types::memory::DocType::Note => "NOTE",
+        };
+        // Truncate long docs to avoid context bloat
+        let content: String = doc.content.chars().take(500).collect();
+        let truncated = if doc.content.len() > 500 { "..." } else { "" };
+        parts.push(format!(
+            "### [{type_label}] {}\n{content}{truncated}\n",
+            doc.title
+        ));
+    }
+
+    parts.join("\n")
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::types::capability::{CapabilityLease, LeaseId};
+    use crate::types::event::ThreadEvent;
+    use crate::types::memory::{DocId, DocType};
+    use crate::types::project::{Project, ProjectId};
+    use crate::types::step::{ActionResult, Step};
+    use crate::types::thread::{Thread, ThreadId, ThreadState};
+
+    struct MockEffects;
+
+    #[async_trait::async_trait]
+    impl EffectExecutor for MockEffects {
+        async fn execute_action(
+            &self,
+            _: &str,
+            _: serde_json::Value,
+            _: &CapabilityLease,
+            _: &crate::traits::effect::ThreadExecutionContext,
+        ) -> Result<ActionResult, EngineError> {
+            Ok(ActionResult {
+                call_id: String::new(),
+                action_name: String::new(),
+                output: serde_json::json!({}),
+                is_error: false,
+                duration: std::time::Duration::from_millis(1),
+            })
+        }
+
+        async fn available_actions(
+            &self,
+            _: &[CapabilityLease],
+        ) -> Result<Vec<ActionDef>, EngineError> {
+            Ok(vec![])
+        }
+    }
+
+    struct DocStore(Vec<MemoryDoc>);
+
+    #[async_trait::async_trait]
+    impl crate::traits::store::Store for DocStore {
+        async fn save_thread(&self, _: &Thread) -> Result<(), EngineError> { Ok(()) }
+        async fn load_thread(&self, _: ThreadId) -> Result<Option<Thread>, EngineError> { Ok(None) }
+        async fn list_threads(&self, _: ProjectId) -> Result<Vec<Thread>, EngineError> { Ok(vec![]) }
+        async fn update_thread_state(&self, _: ThreadId, _: ThreadState) -> Result<(), EngineError> { Ok(()) }
+        async fn save_step(&self, _: &Step) -> Result<(), EngineError> { Ok(()) }
+        async fn load_steps(&self, _: ThreadId) -> Result<Vec<Step>, EngineError> { Ok(vec![]) }
+        async fn append_events(&self, _: &[ThreadEvent]) -> Result<(), EngineError> { Ok(()) }
+        async fn load_events(&self, _: ThreadId) -> Result<Vec<ThreadEvent>, EngineError> { Ok(vec![]) }
+        async fn save_project(&self, _: &Project) -> Result<(), EngineError> { Ok(()) }
+        async fn load_project(&self, _: ProjectId) -> Result<Option<Project>, EngineError> { Ok(None) }
+        async fn save_memory_doc(&self, _: &MemoryDoc) -> Result<(), EngineError> { Ok(()) }
+        async fn load_memory_doc(&self, _: DocId) -> Result<Option<MemoryDoc>, EngineError> { Ok(None) }
+        async fn list_memory_docs(&self, pid: ProjectId) -> Result<Vec<MemoryDoc>, EngineError> {
+            Ok(self.0.iter().filter(|d| d.project_id == pid).cloned().collect())
+        }
+        async fn save_lease(&self, _: &CapabilityLease) -> Result<(), EngineError> { Ok(()) }
+        async fn load_active_leases(&self, _: ThreadId) -> Result<Vec<CapabilityLease>, EngineError> { Ok(vec![]) }
+        async fn revoke_lease(&self, _: LeaseId, _: &str) -> Result<(), EngineError> { Ok(()) }
+    }
+
+    #[tokio::test]
+    async fn context_injects_docs_after_system_prompt() {
+        let project = ProjectId::new();
+        let store: Arc<dyn crate::traits::store::Store> = Arc::new(DocStore(vec![
+            MemoryDoc::new(project, DocType::Lesson, "web tool alias", "Use web-search not web_search"),
+        ]));
+        let retrieval = RetrievalEngine::new(store);
+        let effects: Arc<dyn EffectExecutor> = Arc::new(MockEffects);
+
+        let messages = vec![
+            ThreadMessage::system("You are an assistant."),
+            ThreadMessage::user("search the web"),
+        ];
+
+        let (ctx_msgs, _) = build_step_context(
+            &messages,
+            &[],
+            &effects,
+            Some(&retrieval),
+            project,
+            "search the web",
+        )
+        .await
+        .unwrap();
+
+        // Should have 3 messages: system prompt, injected context, user message
+        assert_eq!(ctx_msgs.len(), 3);
+        assert_eq!(ctx_msgs[0].role, crate::types::message::MessageRole::System);
+        assert_eq!(ctx_msgs[1].role, crate::types::message::MessageRole::System);
+        assert!(ctx_msgs[1].content.contains("Prior Knowledge"));
+        assert!(ctx_msgs[1].content.contains("LESSON"));
+        assert!(ctx_msgs[1].content.contains("web-search"));
+        assert_eq!(ctx_msgs[2].role, crate::types::message::MessageRole::User);
+    }
+
+    #[tokio::test]
+    async fn context_without_retrieval_passes_through() {
+        let effects: Arc<dyn EffectExecutor> = Arc::new(MockEffects);
+        let messages = vec![
+            ThreadMessage::system("prompt"),
+            ThreadMessage::user("hello"),
+        ];
+
+        let (ctx_msgs, _) = build_step_context(
+            &messages,
+            &[],
+            &effects,
+            None,
+            ProjectId::new(),
+            "hello",
+        )
+        .await
+        .unwrap();
+
+        // No injection — same number of messages
+        assert_eq!(ctx_msgs.len(), 2);
+    }
+
+    #[tokio::test]
+    async fn context_no_docs_means_no_injection() {
+        let project = ProjectId::new();
+        let store: Arc<dyn crate::traits::store::Store> = Arc::new(DocStore(vec![]));
+        let retrieval = RetrievalEngine::new(store);
+        let effects: Arc<dyn EffectExecutor> = Arc::new(MockEffects);
+
+        let messages = vec![ThreadMessage::user("hello")];
+
+        let (ctx_msgs, _) = build_step_context(
+            &messages,
+            &[],
+            &effects,
+            Some(&retrieval),
+            project,
+            "hello",
+        )
+        .await
+        .unwrap();
+
+        assert_eq!(ctx_msgs.len(), 1);
+    }
 }

crates/ironclaw_engine/src/executor/loop_engine.rs

@@ -35,6 +35,8 @@ pub struct ExecutionLoop {
     user_id: String,
     /// Optional broadcast sender for live event streaming.
     event_tx: Option<tokio::sync::broadcast::Sender<crate::types::event::ThreadEvent>>,
+    /// Optional retrieval engine for injecting prior knowledge into context.
+    retrieval: Option<crate::memory::RetrievalEngine>,
 }
 
 impl ExecutionLoop {
@@ -56,6 +58,7 @@ impl ExecutionLoop {
             signal_rx,
             user_id,
             event_tx: None,
+            retrieval: None,
         }
     }
 
@@ -68,6 +71,12 @@ impl ExecutionLoop {
         self
     }
 
+    /// Set the retrieval engine for injecting prior knowledge into context.
+    pub fn with_retrieval(mut self, retrieval: crate::memory::RetrievalEngine) -> Self {
+        self.retrieval = Some(retrieval);
+        self
+    }
+
     /// Add an event to the thread and broadcast it for live status updates.
     fn emit_event(&mut self, kind: EventKind) {
         let event = crate::types::event::ThreadEvent::new(self.thread.id, kind);
@@ -187,9 +196,21 @@ impl ExecutionLoop {
             // 4. Get active leases
             let active_leases = self.leases.active_for_thread(self.thread.id).await;
 
-            // 5. Build context
-            let (messages, _actions) =
-                build_step_context(&self.thread.messages, &active_leases, &self.effects).await?;
+            // 5. Build context (inject prior knowledge on first iteration only)
+            let retrieval_ref = if iteration == 0 {
+                self.retrieval.as_ref()
+            } else {
+                None
+            };
+            let (messages, _actions) = build_step_context(
+                &self.thread.messages,
+                &active_leases,
+                &self.effects,
+                retrieval_ref,
+                self.thread.project_id,
+                &self.thread.goal,
+            )
+            .await?;
 
             // 6. Create step
             let mut step = Step::new(self.thread.id, iteration + 1);
@@ -532,11 +553,24 @@ impl ExecutionLoop {
                         output_parts.join("\n")
                     };
                     // Truncate total output to prevent context bloat
-                    let metadata = if output_text.len() > 8000 {
+                    let mut metadata = if output_text.len() > 8000 {
                         format!("[TRUNCATED: last 8000 of {} chars]\n{}", output_text.len(), &output_text[output_text.len()-8000..])
                     } else {
                         output_text
                     };
+
+                    // If code had errors, remind the model about `state`
+                    if code_result.had_error && !persisted_state.as_object().is_some_and(|m| m.is_empty()) {
+                        let keys: Vec<&str> = persisted_state
+                            .as_object()
+                            .map(|m| m.keys().map(String::as_str).collect())
+                            .unwrap_or_default();
+                        metadata.push_str(&format!(
+                            "\n\n[HINT] Variables don't persist between code blocks. Use the `state` dict to access data from previous steps. Available keys: {:?}",
+                            keys
+                        ));
+                    }
+
                     self.thread.add_message(ThreadMessage::system(metadata));
 
                     step.status = StepStatus::Completed;

crates/ironclaw_engine/src/memory/mod.rs

@@ -1,7 +1,7 @@
 //! Memory document system.
 //!
 //! - [`MemoryStore`] — project-scoped document CRUD
-//! - [`RetrievalEngine`] — context building from project docs (Phase 4)
+//! - [`RetrievalEngine`] — context building from project docs via keyword search
 
 pub mod retrieval;
 pub mod store;

crates/ironclaw_engine/src/memory/retrieval.rs

@@ -1,35 +1,298 @@
 //! Context retrieval engine.
 //!
 //! Builds context for thread steps by retrieving relevant memory docs
-//! from the project. Phase 1: stub. Phase 4 implements keyword + semantic search.
+//! from the project. Uses keyword matching against doc title + content,
+//! with priority scoring by doc type (Lessons and Specs rank higher
+//! than Summaries for context injection).
 
+use std::sync::Arc;
+
+use crate::traits::store::Store;
 use crate::types::error::EngineError;
-use crate::types::memory::MemoryDoc;
+use crate::types::memory::{DocType, MemoryDoc};
 use crate::types::project::ProjectId;
 
 /// Retrieves relevant memory docs for a thread's context.
-pub struct RetrievalEngine;
+pub struct RetrievalEngine {
+    store: Arc<dyn Store>,
+}
 
 impl RetrievalEngine {
-    pub fn new() -> Self {
-        Self
+    pub fn new(store: Arc<dyn Store>) -> Self {
+        Self { store }
     }
 
-    /// Retrieve relevant memory docs for the given query.
+    /// Retrieve relevant memory docs for the given query within a project.
     ///
-    /// Phase 1: returns empty vec. Phase 4 implements search.
+    /// Loads all docs for the project, scores them by keyword relevance and
+    /// doc-type priority, and returns the top `max_docs` results.
     pub async fn retrieve_context(
         &self,
-        _project_id: ProjectId,
-        _query: &str,
-        _max_docs: usize,
+        project_id: ProjectId,
+        query: &str,
+        max_docs: usize,
     ) -> Result<Vec<MemoryDoc>, EngineError> {
-        Ok(Vec::new())
+        if max_docs == 0 {
+            return Ok(Vec::new());
+        }
+
+        let all_docs = self.store.list_memory_docs(project_id).await?;
+        if all_docs.is_empty() {
+            return Ok(Vec::new());
+        }
+
+        let keywords = extract_keywords(query);
+        if keywords.is_empty() {
+            // No meaningful keywords — return by doc-type priority alone
+            let mut scored: Vec<(f64, MemoryDoc)> = all_docs
+                .into_iter()
+                .map(|doc| (doc_type_weight(doc.doc_type), doc))
+                .collect();
+            scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
+            scored.truncate(max_docs);
+            return Ok(scored.into_iter().map(|(_, doc)| doc).collect());
+        }
+
+        let mut scored: Vec<(f64, MemoryDoc)> = all_docs
+            .into_iter()
+            .map(|doc| {
+                let keyword_score = keyword_match_score(&doc, &keywords);
+                let type_weight = doc_type_weight(doc.doc_type);
+                // Combined score: keyword relevance (0.0-1.0) + type priority bonus
+                let score = keyword_score + type_weight;
+                (score, doc)
+            })
+            .filter(|(score, _)| *score > 0.0)
+            .collect();
+
+        scored.sort_by(|a, b| b.0.partial_cmp(&a.0).unwrap_or(std::cmp::Ordering::Equal));
+        scored.truncate(max_docs);
+        Ok(scored.into_iter().map(|(_, doc)| doc).collect())
     }
 }
 
-impl Default for RetrievalEngine {
-    fn default() -> Self {
-        Self::new()
+/// Extract lowercase keywords from a query, filtering out stop words.
+fn extract_keywords(query: &str) -> Vec<String> {
+    const STOP_WORDS: &[&str] = &[
+        "a", "an", "the", "is", "are", "was", "were", "be", "been", "being", "have", "has",
+        "had", "do", "does", "did", "will", "would", "could", "should", "may", "might", "shall",
+        "can", "to", "of", "in", "for", "on", "with", "at", "by", "from", "as", "into", "about",
+        "it", "its", "this", "that", "these", "those", "i", "you", "he", "she", "we", "they",
+        "what", "which", "who", "how", "when", "where", "why", "and", "or", "but", "not", "no",
+        "if", "then", "so", "up", "out", "just",
+    ];
+
+    query
+        .split(|c: char| !c.is_alphanumeric() && c != '_' && c != '-')
+        .map(|w| w.to_lowercase())
+        .filter(|w| w.len() >= 2 && !STOP_WORDS.contains(&w.as_str()))
+        .collect()
+}
+
+/// Score how well a doc matches the given keywords (0.0 to 1.0).
+fn keyword_match_score(doc: &MemoryDoc, keywords: &[String]) -> f64 {
+    if keywords.is_empty() {
+        return 0.0;
+    }
+
+    let title_lower = doc.title.to_lowercase();
+    let content_lower = doc.content.to_lowercase();
+
+    let mut matched = 0usize;
+    for kw in keywords {
+        // Title matches are worth more
+        if title_lower.contains(kw.as_str()) {
+            matched += 2;
+        } else if content_lower.contains(kw.as_str()) {
+            matched += 1;
+        }
+    }
+
+    // Normalize: max possible score is keywords.len() * 2 (all in title)
+    let max_score = keywords.len() * 2;
+    matched as f64 / max_score as f64
+}
+
+/// Priority weight by doc type. Higher = more useful for context injection.
+fn doc_type_weight(doc_type: DocType) -> f64 {
+    match doc_type {
+        DocType::Spec => 0.5,     // Missing capability info is highest priority
+        DocType::Lesson => 0.4,   // Lessons prevent repeating mistakes
+        DocType::Playbook => 0.3, // Reusable procedures
+        DocType::Issue => 0.2,    // Known problems
+        DocType::Summary => 0.1,  // Background context
+        DocType::Note => 0.05,    // Scratch notes, lowest priority
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::types::capability::{CapabilityLease, LeaseId};
+    use crate::types::event::ThreadEvent;
+    use crate::types::memory::DocId;
+    use crate::types::project::{Project, ProjectId};
+    use crate::types::step::Step;
+    use crate::types::thread::{Thread, ThreadId, ThreadState};
+
+    /// Mock Store that returns a fixed set of memory docs.
+    struct DocStore {
+        docs: tokio::sync::Mutex<Vec<MemoryDoc>>,
+    }
+
+    impl DocStore {
+        fn new(docs: Vec<MemoryDoc>) -> Arc<Self> {
+            Arc::new(Self {
+                docs: tokio::sync::Mutex::new(docs),
+            })
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl crate::traits::store::Store for DocStore {
+        async fn save_thread(&self, _: &Thread) -> Result<(), EngineError> { Ok(()) }
+        async fn load_thread(&self, _: ThreadId) -> Result<Option<Thread>, EngineError> { Ok(None) }
+        async fn list_threads(&self, _: ProjectId) -> Result<Vec<Thread>, EngineError> { Ok(vec![]) }
+        async fn update_thread_state(&self, _: ThreadId, _: ThreadState) -> Result<(), EngineError> { Ok(()) }
+        async fn save_step(&self, _: &Step) -> Result<(), EngineError> { Ok(()) }
+        async fn load_steps(&self, _: ThreadId) -> Result<Vec<Step>, EngineError> { Ok(vec![]) }
+        async fn append_events(&self, _: &[ThreadEvent]) -> Result<(), EngineError> { Ok(()) }
+        async fn load_events(&self, _: ThreadId) -> Result<Vec<ThreadEvent>, EngineError> { Ok(vec![]) }
+        async fn save_project(&self, _: &Project) -> Result<(), EngineError> { Ok(()) }
+        async fn load_project(&self, _: ProjectId) -> Result<Option<Project>, EngineError> { Ok(None) }
+        async fn save_memory_doc(&self, _: &MemoryDoc) -> Result<(), EngineError> { Ok(()) }
+        async fn load_memory_doc(&self, _: DocId) -> Result<Option<MemoryDoc>, EngineError> { Ok(None) }
+        async fn list_memory_docs(&self, project_id: ProjectId) -> Result<Vec<MemoryDoc>, EngineError> {
+            let docs = self.docs.lock().await;
+            Ok(docs.iter().filter(|d| d.project_id == project_id).cloned().collect())
+        }
+        async fn save_lease(&self, _: &CapabilityLease) -> Result<(), EngineError> { Ok(()) }
+        async fn load_active_leases(&self, _: ThreadId) -> Result<Vec<CapabilityLease>, EngineError> { Ok(vec![]) }
+        async fn revoke_lease(&self, _: LeaseId, _: &str) -> Result<(), EngineError> { Ok(()) }
+    }
+
+    #[test]
+    fn extract_keywords_filters_stop_words() {
+        let kws = extract_keywords("what is the latest news about Iran war");
+        assert!(kws.contains(&"latest".to_string()));
+        assert!(kws.contains(&"news".to_string()));
+        assert!(kws.contains(&"iran".to_string()));
+        assert!(kws.contains(&"war".to_string()));
+        assert!(!kws.contains(&"the".to_string()));
+        assert!(!kws.contains(&"is".to_string()));
+    }
+
+    #[test]
+    fn extract_keywords_handles_special_chars() {
+        let kws = extract_keywords("web_search web-fetch tool");
+        assert!(kws.contains(&"web_search".to_string()));
+        assert!(kws.contains(&"web-fetch".to_string()));
+        assert!(kws.contains(&"tool".to_string()));
+    }
+
+    #[test]
+    fn keyword_match_title_beats_content() {
+        use crate::types::project::ProjectId;
+
+        let doc = MemoryDoc::new(
+            ProjectId::new(),
+            DocType::Lesson,
+            "Lesson about web_search errors",
+            "The tool was not found during execution.",
+        );
+
+        let keywords = vec!["web_search".to_string()];
+        let score = keyword_match_score(&doc, &keywords);
+        // Title match = 2/2 = 1.0
+        assert!((score - 1.0).abs() < f64::EPSILON);
+
+        let keywords2 = vec!["execution".to_string()];
+        let score2 = keyword_match_score(&doc, &keywords2);
+        // Content-only match = 1/2 = 0.5
+        assert!((score2 - 0.5).abs() < f64::EPSILON);
+    }
+
+    #[test]
+    fn doc_type_weight_ordering() {
+        assert!(doc_type_weight(DocType::Spec) > doc_type_weight(DocType::Lesson));
+        assert!(doc_type_weight(DocType::Lesson) > doc_type_weight(DocType::Playbook));
+        assert!(doc_type_weight(DocType::Playbook) > doc_type_weight(DocType::Issue));
+        assert!(doc_type_weight(DocType::Issue) > doc_type_weight(DocType::Summary));
+        assert!(doc_type_weight(DocType::Summary) > doc_type_weight(DocType::Note));
+    }
+
+    #[tokio::test]
+    async fn retrieve_returns_relevant_docs_by_keyword() {
+        let project = ProjectId::new();
+        let store = DocStore::new(vec![
+            MemoryDoc::new(project, DocType::Lesson, "web_search tool alias", "Use web-search not web_search"),
+            MemoryDoc::new(project, DocType::Summary, "weather query", "Fetched weather data"),
+            MemoryDoc::new(project, DocType::Issue, "API timeout", "External API timed out"),
+        ]);
+        let engine = RetrievalEngine::new(store);
+
+        let docs = engine.retrieve_context(project, "web_search error", 5).await.unwrap();
+        assert!(!docs.is_empty());
+        // The lesson about web_search should rank first (keyword + type weight)
+        assert_eq!(docs[0].doc_type, DocType::Lesson);
+        assert!(docs[0].title.contains("web_search"));
+    }
+
+    #[tokio::test]
+    async fn retrieve_respects_project_scoping() {
+        let project_a = ProjectId::new();
+        let project_b = ProjectId::new();
+        let store = DocStore::new(vec![
+            MemoryDoc::new(project_a, DocType::Lesson, "Lesson for project A", "Some lesson"),
+            MemoryDoc::new(project_b, DocType::Lesson, "Lesson for project B", "Other lesson"),
+        ]);
+        let engine = RetrievalEngine::new(store);
+
+        let docs_a = engine.retrieve_context(project_a, "lesson", 5).await.unwrap();
+        assert_eq!(docs_a.len(), 1);
+        assert!(docs_a[0].title.contains("project A"));
+
+        let docs_b = engine.retrieve_context(project_b, "lesson", 5).await.unwrap();
+        assert_eq!(docs_b.len(), 1);
+        assert!(docs_b[0].title.contains("project B"));
+    }
+
+    #[tokio::test]
+    async fn retrieve_respects_max_docs_limit() {
+        let project = ProjectId::new();
+        let store = DocStore::new(vec![
+            MemoryDoc::new(project, DocType::Lesson, "Lesson 1", "Content 1"),
+            MemoryDoc::new(project, DocType::Lesson, "Lesson 2", "Content 2"),
+            MemoryDoc::new(project, DocType::Lesson, "Lesson 3", "Content 3"),
+        ]);
+        let engine = RetrievalEngine::new(store);
+
+        let docs = engine.retrieve_context(project, "lesson", 2).await.unwrap();
+        assert_eq!(docs.len(), 2);
+    }
+
+    #[tokio::test]
+    async fn retrieve_empty_store_returns_empty() {
+        let project = ProjectId::new();
+        let store = DocStore::new(vec![]);
+        let engine = RetrievalEngine::new(store);
+
+        let docs = engine.retrieve_context(project, "anything", 5).await.unwrap();
+        assert!(docs.is_empty());
+    }
+
+    #[tokio::test]
+    async fn retrieve_spec_ranks_above_summary() {
+        let project = ProjectId::new();
+        let store = DocStore::new(vec![
+            MemoryDoc::new(project, DocType::Summary, "Summary of search", "searched the web"),
+            MemoryDoc::new(project, DocType::Spec, "Missing search tool", "ALIAS: web_search -> web-search"),
+        ]);
+        let engine = RetrievalEngine::new(store);
+
+        let docs = engine.retrieve_context(project, "search", 5).await.unwrap();
+        assert_eq!(docs.len(), 2);
+        // Spec should rank first due to higher type weight
+        assert_eq!(docs[0].doc_type, DocType::Spec);
     }
 }

crates/ironclaw_engine/src/reflection/mod.rs

@@ -5,6 +5,8 @@
 //! - Summary — what the thread accomplished
 //! - Lesson — what was learned from errors/workarounds
 //! - Issue — unresolved problems for follow-up
+//! - Spec — missing capabilities / tool alias suggestions
+//! - Playbook — reusable multi-step procedures from successful threads
 
 pub mod pipeline;
 

crates/ironclaw_engine/src/reflection/pipeline.rs

@@ -77,6 +77,42 @@ pub async fn reflect(
         total_tokens.output_tokens += tokens.output_tokens;
     }
 
+    // 4. Missing capabilities (if tool-not-found errors detected)
+    let has_missing_tools = thread.events.iter().any(|e| {
+        if let EventKind::ActionFailed { error, .. } = &e.kind {
+            error.contains("not found") || error.contains("not available")
+        } else {
+            false
+        }
+    });
+    if has_missing_tools {
+        let (spec_doc, tokens) =
+            produce_doc(thread, llm, DocType::Spec, &transcript, SPEC_PROMPT).await?;
+        if spec_doc.content.len() > 20 {
+            docs.push(spec_doc);
+        }
+        total_tokens.input_tokens += tokens.input_tokens;
+        total_tokens.output_tokens += tokens.output_tokens;
+    }
+
+    // 5. Playbook (successful threads with multiple tool-using steps)
+    let action_count = thread
+        .events
+        .iter()
+        .filter(|e| matches!(e.kind, EventKind::ActionExecuted { .. }))
+        .count();
+    let thread_succeeded =
+        thread.state == crate::types::thread::ThreadState::Completed && !thread_failed;
+    if thread_succeeded && action_count >= 2 {
+        let (playbook_doc, tokens) =
+            produce_doc(thread, llm, DocType::Playbook, &transcript, PLAYBOOK_PROMPT).await?;
+        if playbook_doc.content.len() > 20 {
+            docs.push(playbook_doc);
+        }
+        total_tokens.input_tokens += tokens.input_tokens;
+        total_tokens.output_tokens += tokens.output_tokens;
+    }
+
     debug!(
         thread_id = %thread.id,
         docs_produced = docs.len(),
@@ -115,6 +151,21 @@ Identify any unresolved issues from this thread. Focus on:
 - Data quality issues encountered
 If there are no unresolved issues, write 'No issues.'.";
 
+const SPEC_PROMPT: &str = "\
+This thread encountered missing tools or capabilities. Analyze the errors and identify:
+- Which tool names were attempted but not found
+- What the correct tool name might be (if a similar tool exists under a different name)
+- What capabilities would need to be added to handle this task
+For each missing capability, write one line: MISSING: <attempted_name> -> <suggestion or description>.
+If the tool exists under a different name, write: ALIAS: <attempted_name> -> <correct_name>.";
+
+const PLAYBOOK_PROMPT: &str = "\
+This thread successfully completed a multi-step task. Extract a reusable playbook:
+- List the steps taken in order (tool calls, queries, transformations)
+- Note which tools were used and in what sequence
+- Describe the pattern so it can be reused for similar tasks
+Write the playbook as a numbered list of steps. Be specific about tool names and parameters used.";
+
 // ── Helpers ─────────────────────────────────────────────────
 
 /// Build a concise transcript of the thread's work.
@@ -208,3 +259,198 @@ async fn produce_doc(
 
     Ok((doc, output.usage))
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::traits::llm::{LlmCallConfig, LlmOutput};
+    use crate::types::capability::ActionDef;
+    use crate::types::event::ThreadEvent;
+    use crate::types::project::ProjectId;
+    use crate::types::step::TokenUsage;
+    use crate::types::thread::{ThreadConfig, ThreadType};
+    use std::sync::Mutex;
+
+    struct MockLlm {
+        responses: Mutex<Vec<String>>,
+    }
+
+    impl MockLlm {
+        fn with_responses(responses: Vec<&str>) -> Arc<dyn crate::traits::llm::LlmBackend> {
+            Arc::new(Self {
+                responses: Mutex::new(responses.into_iter().map(String::from).collect()),
+            })
+        }
+    }
+
+    #[async_trait::async_trait]
+    impl crate::traits::llm::LlmBackend for MockLlm {
+        async fn complete(
+            &self,
+            _: &[ThreadMessage],
+            _: &[ActionDef],
+            _: &LlmCallConfig,
+        ) -> Result<LlmOutput, EngineError> {
+            let mut r = self.responses.lock().unwrap();
+            let text = if r.is_empty() {
+                "mock response".to_string()
+            } else {
+                r.remove(0)
+            };
+            Ok(LlmOutput {
+                response: LlmResponse::Text(text),
+                usage: TokenUsage {
+                    input_tokens: 100,
+                    output_tokens: 50,
+                    ..TokenUsage::default()
+                },
+            })
+        }
+
+        fn model_name(&self) -> &str {
+            "mock"
+        }
+    }
+
+    fn make_completed_thread() -> Thread {
+        let mut thread = Thread::new(
+            "test task",
+            ThreadType::Foreground,
+            ProjectId::new(),
+            ThreadConfig::default(),
+        );
+        thread.state = crate::types::thread::ThreadState::Completed;
+        thread
+    }
+
+    #[tokio::test]
+    async fn reflect_produces_summary_for_clean_thread() {
+        let thread = make_completed_thread();
+        let llm = MockLlm::with_responses(vec!["Thread accomplished the test task successfully."]);
+
+        let result = reflect(&thread, &llm).await.unwrap();
+        assert_eq!(result.docs.len(), 1);
+        assert_eq!(result.docs[0].doc_type, DocType::Summary);
+    }
+
+    #[tokio::test]
+    async fn reflect_produces_lesson_on_errors() {
+        let mut thread = make_completed_thread();
+        thread.events.push(ThreadEvent::new(
+            thread.id,
+            EventKind::ActionFailed {
+                step_id: crate::types::step::StepId::new(),
+                action_name: "web_search".into(),
+                call_id: String::new(),
+                error: "Tool web_search not found".into(),
+            },
+        ));
+
+        let llm = MockLlm::with_responses(vec![
+            "Summary of thread with errors.",
+            "Lesson: use web-search instead of web_search.",
+            "Issue: web_search tool is missing.",
+            "ALIAS: web_search -> web-search",
+        ]);
+
+        let result = reflect(&thread, &llm).await.unwrap();
+        let types: Vec<DocType> = result.docs.iter().map(|d| d.doc_type).collect();
+        assert!(types.contains(&DocType::Summary));
+        assert!(types.contains(&DocType::Lesson));
+        assert!(types.contains(&DocType::Issue));
+        assert!(types.contains(&DocType::Spec));
+    }
+
+    #[tokio::test]
+    async fn reflect_produces_spec_on_tool_not_found() {
+        let mut thread = make_completed_thread();
+        thread.events.push(ThreadEvent::new(
+            thread.id,
+            EventKind::ActionFailed {
+                step_id: crate::types::step::StepId::new(),
+                action_name: "missing_tool".into(),
+                call_id: String::new(),
+                error: "Tool missing_tool not found".into(),
+            },
+        ));
+
+        let llm = MockLlm::with_responses(vec![
+            "Summary.",
+            "Lesson learned.",
+            "Issues found.",
+            "MISSING: missing_tool -> needs implementation",
+        ]);
+
+        let result = reflect(&thread, &llm).await.unwrap();
+        let spec_docs: Vec<&MemoryDoc> = result
+            .docs
+            .iter()
+            .filter(|d| d.doc_type == DocType::Spec)
+            .collect();
+        assert_eq!(spec_docs.len(), 1);
+        assert!(spec_docs[0].content.contains("MISSING"));
+    }
+
+    #[tokio::test]
+    async fn reflect_produces_playbook_on_successful_multi_step() {
+        let mut thread = make_completed_thread();
+        // Add 2+ action executed events to trigger playbook
+        thread.events.push(ThreadEvent::new(
+            thread.id,
+            EventKind::ActionExecuted {
+                step_id: crate::types::step::StepId::new(),
+                action_name: "web-search".into(),
+                call_id: String::new(),
+                duration_ms: 100,
+            },
+        ));
+        thread.events.push(ThreadEvent::new(
+            thread.id,
+            EventKind::ActionExecuted {
+                step_id: crate::types::step::StepId::new(),
+                action_name: "llm_query".into(),
+                call_id: String::new(),
+                duration_ms: 200,
+            },
+        ));
+
+        let llm = MockLlm::with_responses(vec![
+            "Summary of successful thread.",
+            "1. Search web for topic\n2. Analyze results with llm_query\n3. Return summary",
+        ]);
+
+        let result = reflect(&thread, &llm).await.unwrap();
+        let playbook_docs: Vec<&MemoryDoc> = result
+            .docs
+            .iter()
+            .filter(|d| d.doc_type == DocType::Playbook)
+            .collect();
+        assert_eq!(playbook_docs.len(), 1);
+        assert!(playbook_docs[0].title.starts_with("Playbook:"));
+    }
+
+    #[tokio::test]
+    async fn reflect_skips_playbook_for_single_action() {
+        let mut thread = make_completed_thread();
+        // Only 1 action — not enough for a playbook
+        thread.events.push(ThreadEvent::new(
+            thread.id,
+            EventKind::ActionExecuted {
+                step_id: crate::types::step::StepId::new(),
+                action_name: "echo".into(),
+                call_id: String::new(),
+                duration_ms: 5,
+            },
+        ));
+
+        let llm = MockLlm::with_responses(vec!["Simple summary."]);
+
+        let result = reflect(&thread, &llm).await.unwrap();
+        let playbook_docs: Vec<&MemoryDoc> = result
+            .docs
+            .iter()
+            .filter(|d| d.doc_type == DocType::Playbook)
+            .collect();
+        assert!(playbook_docs.is_empty());
+    }
+}

crates/ironclaw_engine/src/runtime/manager.rs

@@ -151,8 +151,12 @@ impl ThreadManager {
         let leases = Arc::clone(&self.leases);
         let policy = Arc::clone(&self.policy);
 
+        let store_for_retrieval = Arc::clone(&self.store);
+        let retrieval = crate::memory::RetrievalEngine::new(store_for_retrieval);
+
         let exec_loop = ExecutionLoop::new(thread, llm, effects, leases, policy, rx, user_id)
-            .with_event_tx(self.event_tx.clone());
+            .with_event_tx(self.event_tx.clone())
+            .with_retrieval(retrieval);
 
         // Spawn background task
         let store_for_task = Arc::clone(&self.store);