feat: multi-tenant auth with per-user scoping

Multi-user authentication and authorization for IronClaw gateway:
- Token-based auth mapping tokens to user IDs via GATEWAY_USER_TOKENS
- Per-user SSE broadcast scoping
- Per-user rate limiting with poisoned lock recovery
- Handler auth and ownership checks for jobs, settings, routines
- Extension secrets scoped per-user
- Chat handlers use authenticated identity
- Reverse proxy deployment documentation
- Comprehensive integration tests for auth, SSE, rate limiting, and job isolation

tests/e2e_advanced_traces.rs

@@ -661,7 +661,7 @@ mod advanced {
             .await
             .expect("failed to inject test token");
 
-        let activate_result = ext_mgr.activate("mock-notion").await;
+        let activate_result = ext_mgr.activate("mock-notion", "default").await;
         assert!(
             activate_result.is_ok(),
             "activation failed: {:?}",

tests/module_init_integration.rs

@@ -216,7 +216,7 @@ async fn extension_manager_with_process_manager_constructs() {
     );
 
     // Verify the manager is functional — list returns Ok.
-    let result = manager.list(None, false).await;
+    let result = manager.list(None, false, "test").await;
     assert!(result.is_ok(), "list should succeed on empty manager");
     assert!(result.unwrap().is_empty());
 }

tests/multi_tenant_system_prompt.rs

@@ -0,0 +1,254 @@
+//! Tests proving that multi-tenant system prompts are broken.
+//!
+//! Bug: In multi-tenant mode, the agent loop uses `self.workspace()` which
+//! returns a single shared workspace (user_id="default"). Identity files
+//! (IDENTITY.md, SOUL.md, USER.md) seeded under per-user IDs ("alice",
+//! "bob") are invisible to this workspace, so the system prompt is
+//! empty/wrong.
+//!
+//! These tests:
+//! 1. Seed identity files for two users (alice, bob) in the database
+//! 2. Send messages as each user
+//! 3. Verify the system prompt in captured LLM requests contains the
+//!    correct user's identity
+//! 4. Verify user A's identity doesn't leak into user B's prompt
+//!
+//! All tests are expected to FAIL until the bug is fixed.
+
+#[cfg(feature = "libsql")]
+mod support;
+
+#[cfg(feature = "libsql")]
+mod tests {
+    use std::sync::Arc;
+    use std::time::Duration;
+
+    use ironclaw::channels::IncomingMessage;
+    use ironclaw::llm::Role;
+    use ironclaw::workspace::Workspace;
+
+    use crate::support::test_rig::TestRigBuilder;
+    use crate::support::trace_llm::{LlmTrace, TraceResponse, TraceStep};
+
+    const TIMEOUT: Duration = Duration::from_secs(15);
+
+    const ALICE_USER_ID: &str = "alice";
+    const BOB_USER_ID: &str = "bob";
+
+    const ALICE_IDENTITY: &str = "You are Alice's personal assistant. \
+        Alice is a software engineer who lives in Seattle.";
+    const BOB_IDENTITY: &str = "You are Bob's personal assistant. \
+        Bob is a marine biologist who lives in Miami.";
+
+    /// Create a simple trace that returns a canned text response.
+    /// We need one step per message we plan to send.
+    fn simple_trace(num_steps: usize) -> LlmTrace {
+        let steps: Vec<TraceStep> = (0..num_steps)
+            .map(|i| TraceStep {
+                request_hint: None,
+                response: TraceResponse::Text {
+                    content: format!("Response {}", i),
+                    input_tokens: 100,
+                    output_tokens: 10,
+                },
+                expected_tool_results: Vec::new(),
+            })
+            .collect();
+
+        // Create separate turns for each step so the trace replays correctly.
+        let turns: Vec<crate::support::trace_llm::TraceTurn> = steps
+            .into_iter()
+            .enumerate()
+            .map(|(i, step)| crate::support::trace_llm::TraceTurn {
+                user_input: format!("message {}", i),
+                steps: vec![step],
+                expects: Default::default(),
+            })
+            .collect();
+
+        LlmTrace::new("test-model", turns)
+    }
+
+    /// Seed identity files for a user by creating a workspace scoped to that
+    /// user and writing IDENTITY.md.
+    async fn seed_identity(db: &Arc<dyn ironclaw::db::Database>, user_id: &str, content: &str) {
+        let ws = Workspace::new_with_db(user_id, db.clone());
+        ws.write("IDENTITY.md", content)
+            .await
+            .unwrap_or_else(|e| panic!("Failed to seed IDENTITY.md for {user_id}: {e}"));
+    }
+
+    /// Extract the system prompt from captured LLM requests.
+    ///
+    /// The system prompt is the first message with role=System in the first
+    /// LLM request for a given turn.
+    fn extract_system_prompt(requests: &[Vec<ironclaw::llm::ChatMessage>]) -> Option<String> {
+        requests.last().and_then(|msgs| {
+            msgs.iter()
+                .find(|m| matches!(m.role, Role::System))
+                .map(|m| m.content.clone())
+        })
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 1: Alice's identity should appear in system prompt when messaging
+    // as Alice.
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn alice_system_prompt_contains_alice_identity() {
+        let trace = simple_trace(1);
+        let rig = TestRigBuilder::new()
+            .with_trace(trace)
+            .build()
+            .await;
+
+        // Seed alice's identity into the database
+        let db = rig.database();
+        seed_identity(db, ALICE_USER_ID, ALICE_IDENTITY).await;
+
+        // Send a message AS alice (using her user_id)
+        let msg = IncomingMessage::new("test", ALICE_USER_ID, "Hello, who am I?");
+        rig.send_incoming(msg).await;
+        let _responses = rig.wait_for_responses(1, TIMEOUT).await;
+
+        // The system prompt sent to the LLM should contain Alice's identity
+        let requests = rig.captured_llm_requests();
+        let system_prompt = extract_system_prompt(&requests)
+            .expect("Expected a system prompt in the LLM request");
+
+        assert!(
+            system_prompt.contains("Alice is a software engineer"),
+            "System prompt should contain Alice's identity when messaging as Alice.\n\
+             Actual system prompt:\n{system_prompt}"
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 2: Bob's identity should appear in system prompt when messaging
+    // as Bob.
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn bob_system_prompt_contains_bob_identity() {
+        let trace = simple_trace(1);
+        let rig = TestRigBuilder::new()
+            .with_trace(trace)
+            .build()
+            .await;
+
+        // Seed bob's identity into the database
+        let db = rig.database();
+        seed_identity(db, BOB_USER_ID, BOB_IDENTITY).await;
+
+        // Send a message AS bob
+        let msg = IncomingMessage::new("test", BOB_USER_ID, "Hello, who am I?");
+        rig.send_incoming(msg).await;
+        let _responses = rig.wait_for_responses(1, TIMEOUT).await;
+
+        // The system prompt should contain Bob's identity
+        let requests = rig.captured_llm_requests();
+        let system_prompt = extract_system_prompt(&requests)
+            .expect("Expected a system prompt in the LLM request");
+
+        assert!(
+            system_prompt.contains("Bob is a marine biologist"),
+            "System prompt should contain Bob's identity when messaging as Bob.\n\
+             Actual system prompt:\n{system_prompt}"
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 3: Alice's identity must NOT appear in Bob's system prompt.
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn alice_identity_does_not_leak_into_bob_prompt() {
+        let trace = simple_trace(1);
+        let rig = TestRigBuilder::new()
+            .with_trace(trace)
+            .build()
+            .await;
+
+        // Seed BOTH users' identities
+        let db = rig.database();
+        seed_identity(db, ALICE_USER_ID, ALICE_IDENTITY).await;
+        seed_identity(db, BOB_USER_ID, BOB_IDENTITY).await;
+
+        // Send a message AS bob
+        let msg = IncomingMessage::new("test", BOB_USER_ID, "Tell me about myself");
+        rig.send_incoming(msg).await;
+        let _responses = rig.wait_for_responses(1, TIMEOUT).await;
+
+        // Bob's prompt must NOT contain Alice's identity
+        let requests = rig.captured_llm_requests();
+        let system_prompt = extract_system_prompt(&requests);
+
+        if let Some(ref prompt) = system_prompt {
+            assert!(
+                !prompt.contains("Alice is a software engineer"),
+                "Alice's identity LEAKED into Bob's system prompt!\n\
+                 System prompt:\n{prompt}"
+            );
+        }
+        // Also verify Bob's identity IS present (compound check)
+        let prompt = system_prompt
+            .expect("Expected a system prompt in the LLM request");
+        assert!(
+            prompt.contains("Bob is a marine biologist"),
+            "Bob's own identity should be in his system prompt.\n\
+             Actual system prompt:\n{prompt}"
+        );
+
+        rig.shutdown();
+    }
+
+    // -----------------------------------------------------------------------
+    // Test 4: Bob's identity must NOT appear in Alice's system prompt.
+    // -----------------------------------------------------------------------
+
+    #[tokio::test]
+    async fn bob_identity_does_not_leak_into_alice_prompt() {
+        let trace = simple_trace(1);
+        let rig = TestRigBuilder::new()
+            .with_trace(trace)
+            .build()
+            .await;
+
+        // Seed BOTH users' identities
+        let db = rig.database();
+        seed_identity(db, ALICE_USER_ID, ALICE_IDENTITY).await;
+        seed_identity(db, BOB_USER_ID, BOB_IDENTITY).await;
+
+        // Send a message AS alice
+        let msg = IncomingMessage::new("test", ALICE_USER_ID, "Tell me about myself");
+        rig.send_incoming(msg).await;
+        let _responses = rig.wait_for_responses(1, TIMEOUT).await;
+
+        // Alice's prompt must NOT contain Bob's identity
+        let requests = rig.captured_llm_requests();
+        let system_prompt = extract_system_prompt(&requests);
+
+        if let Some(ref prompt) = system_prompt {
+            assert!(
+                !prompt.contains("Bob is a marine biologist"),
+                "Bob's identity LEAKED into Alice's system prompt!\n\
+                 System prompt:\n{prompt}"
+            );
+        }
+        // Also verify Alice's identity IS present
+        let prompt = system_prompt
+            .expect("Expected a system prompt in the LLM request");
+        assert!(
+            prompt.contains("Alice is a software engineer"),
+            "Alice's own identity should be in her system prompt.\n\
+             Actual system prompt:\n{prompt}"
+        );
+
+        rig.shutdown();
+    }
+}

tests/openai_compat_integration.rs

@@ -191,8 +191,9 @@ async fn start_test_server_with_provider(
 ) -> (SocketAddr, Arc<GatewayState>) {
     let state = Arc::new(GatewayState {
         msg_tx: tokio::sync::RwLock::new(None),
-        sse: SseManager::new(),
+        sse: Arc::new(SseManager::new()),
         workspace: None,
+        workspace_pool: None,
         session_manager: None,
         log_broadcaster: None,
         log_level_handle: None,
@@ -202,13 +203,13 @@ async fn start_test_server_with_provider(
         job_manager: None,
         prompt_queue: None,
         scheduler: None,
-        user_id: "test-user".to_string(),
+        default_user_id: "test-user".to_string(),
         shutdown_tx: tokio::sync::RwLock::new(None),
         ws_tracker: Some(Arc::new(WsConnectionTracker::new())),
         llm_provider: Some(llm_provider),
         skill_registry: None,
         skill_catalog: None,
-        chat_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(30, 60),
+        chat_rate_limiter: ironclaw::channels::web::server::PerUserRateLimiter::new(30, 60),
         oauth_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(10, 60),
         webhook_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(10, 60),
         registry_entries: Vec::new(),
@@ -218,8 +219,12 @@ async fn start_test_server_with_provider(
         active_config: ironclaw::channels::web::server::ActiveConfigSnapshot::default(),
     });
 
+    let auth = ironclaw::channels::web::auth::MultiAuthState::single(
+        AUTH_TOKEN.to_string(),
+        "test-user".to_string(),
+    );
     let addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
-    let bound_addr = start_server(addr, state.clone(), AUTH_TOKEN.to_string())
+    let bound_addr = start_server(addr, state.clone(), auth)
         .await
         .expect("Failed to start test server");
 
@@ -684,8 +689,9 @@ async fn test_no_llm_provider_returns_503() {
     // Create state WITHOUT llm_provider
     let state = Arc::new(GatewayState {
         msg_tx: tokio::sync::RwLock::new(None),
-        sse: SseManager::new(),
+        sse: Arc::new(SseManager::new()),
         workspace: None,
+        workspace_pool: None,
         session_manager: None,
         log_broadcaster: None,
         log_level_handle: None,
@@ -695,13 +701,13 @@ async fn test_no_llm_provider_returns_503() {
         job_manager: None,
         prompt_queue: None,
         scheduler: None,
-        user_id: "test-user".to_string(),
+        default_user_id: "test-user".to_string(),
         shutdown_tx: tokio::sync::RwLock::new(None),
         ws_tracker: Some(Arc::new(WsConnectionTracker::new())),
         llm_provider: None, // No LLM!
         skill_registry: None,
         skill_catalog: None,
-        chat_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(30, 60),
+        chat_rate_limiter: ironclaw::channels::web::server::PerUserRateLimiter::new(30, 60),
         oauth_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(10, 60),
         webhook_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(10, 60),
         registry_entries: Vec::new(),
@@ -711,8 +717,12 @@ async fn test_no_llm_provider_returns_503() {
         active_config: ironclaw::channels::web::server::ActiveConfigSnapshot::default(),
     });
 
+    let auth = ironclaw::channels::web::auth::MultiAuthState::single(
+        AUTH_TOKEN.to_string(),
+        "test-user".to_string(),
+    );
     let addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
-    let bound_addr = start_server(addr, state, AUTH_TOKEN.to_string())
+    let bound_addr = start_server(addr, state, auth)
         .await
         .unwrap();
 
@@ -741,9 +751,10 @@ async fn test_chat_completions_body_too_large() {
     let state = ironclaw::channels::web::test_helpers::TestGatewayBuilder::new()
         .llm_provider(llm_provider)
         .build();
-    let auth_state = ironclaw::channels::web::auth::AuthState {
-        token: AUTH_TOKEN.to_string(),
-    };
+    let auth_state = ironclaw::channels::web::auth::MultiAuthState::single(
+        AUTH_TOKEN.to_string(),
+        "test-user".to_string(),
+    );
 
     let app = Router::new()
         .route(

tests/support/gateway_workflow_harness.rs

@@ -14,7 +14,8 @@ use ironclaw::agent::{Agent, AgentDeps, SessionManager as AgentSessionManager};
 use ironclaw::app::{AppBuilder, AppBuilderFlags};
 use ironclaw::channels::IncomingMessage;
 use ironclaw::channels::web::log_layer::LogBroadcaster;
-use ironclaw::channels::web::server::{GatewayState, RateLimiter, start_server};
+use ironclaw::channels::web::auth::MultiAuthState;
+use ironclaw::channels::web::server::{GatewayState, PerUserRateLimiter, RateLimiter, start_server};
 use ironclaw::channels::web::sse::SseManager;
 use ironclaw::channels::web::ws::WsConnectionTracker;
 use ironclaw::config::{Config, RegistryProviderConfig, RoutineConfig};
@@ -211,8 +212,9 @@ impl GatewayWorkflowHarness {
 
         let gateway_state = Arc::new(GatewayState {
             msg_tx: tokio::sync::RwLock::new(Some(gw_tx)),
-            sse: SseManager::new(),
+            sse: Arc::new(SseManager::new()),
             workspace: components.workspace.clone(),
+            workspace_pool: None,
             session_manager: Some(Arc::clone(&agent_session_manager)),
             log_broadcaster: None,
             log_level_handle: None,
@@ -222,13 +224,13 @@ impl GatewayWorkflowHarness {
             job_manager: None,
             prompt_queue: None,
             scheduler: Some(scheduler_slot.clone()),
-            user_id: user_id.clone(),
+            default_user_id: user_id.clone(),
             shutdown_tx: tokio::sync::RwLock::new(None),
             ws_tracker: Some(Arc::new(WsConnectionTracker::new())),
             llm_provider: Some(Arc::clone(&components.llm)),
             skill_registry: components.skill_registry.clone(),
             skill_catalog: components.skill_catalog.clone(),
-            chat_rate_limiter: RateLimiter::new(120, 60),
+            chat_rate_limiter: PerUserRateLimiter::new(120, 60),
             oauth_rate_limiter: RateLimiter::new(10, 60),
             webhook_rate_limiter: RateLimiter::new(10, 60),
             registry_entries: Vec::new(),
@@ -254,7 +256,7 @@ impl GatewayWorkflowHarness {
                 skills_config: components.config.skills.clone(),
                 hooks: components.hooks,
                 cost_guard: components.cost_guard,
-                sse_tx: Some(gateway_state.sse.sender()),
+                sse_tx: None,
                 http_interceptor: None,
                 transcription: None,
                 document_extraction: None,
@@ -288,10 +290,11 @@ impl GatewayWorkflowHarness {
         }
 
         let auth_token = "gateway-test-token".to_string();
+        let auth = MultiAuthState::single(auth_token.clone(), user_id.clone());
         let addr = start_server(
             "127.0.0.1:0".parse().expect("valid localhost addr"),
             Arc::clone(&gateway_state),
-            auth_token.clone(),
+            auth,
         )
         .await
         .expect("failed to start gateway server");

tests/ws_gateway_integration.rs

@@ -39,8 +39,9 @@ async fn start_test_server() -> (
 
     let state = Arc::new(GatewayState {
         msg_tx: tokio::sync::RwLock::new(Some(agent_tx)),
-        sse: SseManager::new(),
+        sse: Arc::new(SseManager::new()),
         workspace: None,
+        workspace_pool: None,
         session_manager: None,
         log_broadcaster: None,
         log_level_handle: None,
@@ -50,13 +51,13 @@ async fn start_test_server() -> (
         job_manager: None,
         prompt_queue: None,
         scheduler: None,
-        user_id: "test-user".to_string(),
+        default_user_id: "test-user".to_string(),
         shutdown_tx: tokio::sync::RwLock::new(None),
         ws_tracker: Some(Arc::new(WsConnectionTracker::new())),
         llm_provider: None,
         skill_registry: None,
         skill_catalog: None,
-        chat_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(30, 60),
+        chat_rate_limiter: ironclaw::channels::web::server::PerUserRateLimiter::new(30, 60),
         oauth_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(10, 60),
         webhook_rate_limiter: ironclaw::channels::web::server::RateLimiter::new(10, 60),
         registry_entries: Vec::new(),
@@ -66,8 +67,12 @@ async fn start_test_server() -> (
         active_config: ironclaw::channels::web::server::ActiveConfigSnapshot::default(),
     });
 
+    let auth = ironclaw::channels::web::auth::MultiAuthState::single(
+        AUTH_TOKEN.to_string(),
+        "test-user".to_string(),
+    );
     let addr: SocketAddr = "127.0.0.1:0".parse().unwrap();
-    let bound_addr = start_server(addr, state.clone(), AUTH_TOKEN.to_string())
+    let bound_addr = start_server(addr, state.clone(), auth)
         .await
         .expect("Failed to start test server");