mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-31 16:49:34 +00:00
fix: prevent session lock contention blocking message processing (#783)
* fix: prevent session lock contention blocking message processing ## Problem After container restart, POST /api/chat/send returns 202 ACCEPTED but messages don't appear in conversation_messages and agent never responds. Messages get stuck in "stale state" after restart. Root cause: Session lock was held for entire duration of chat_threads_handler and chat_history_handler, including during slow database queries. This blocked the agent loop from acquiring the session lock to process incoming messages, causing them to hang indefinitely. ## Solution 1. **Release session lock early in chat_threads_handler**: Only acquire lock when reading active_thread at response time, not during DB queries for thread list. DB operations no longer block message processing. 2. **Release session lock early in chat_history_handler**: Only acquire lock when accessing in-memory thread state, not during paginated DB queries or thread ownership checks. DB operations no longer block message processing. 3. **Add comprehensive logging**: Track message flow from receipt through session resolution, thread hydration, and state transitions. Helps diagnose future issues: - Message queued to agent loop (chat_send_handler) - Processing message from channel (handle_message) - Hydrating thread from DB (maybe_hydrate_thread) - Resolving session and thread (resolve_thread) - Checking thread state (process_user_input) - Persisting user message (persist_user_message) ## Impact - Message processing no longer blocks on session lock contention - API response times for thread list/history queries unaffected (DB queries still happen, but lock is not held) - Better diagnostics for future debugging ## Testing - All 2756 tests pass - Code compiles with zero clippy warnings - No changes to user-facing API or behavior, only lock timing Co-Authored-By: Claude Haiku 4.5 <[email protected]> * security: redact PII from info-level logs Downgrade user_id and channel logging to debug level to prevent exposing Personally Identifiable Information (PII) in production logs. The user_id field can contain sensitive information such as phone numbers (e.g., for Signal messages). Logging PII in cleartext at the info level creates a security and privacy risk, as these logs may be stored in persistent storage, indexed by log management systems, or accessible to unauthorized personnel. Changes: - Info level: logs only message_id (UUID) for tracking - Debug level: logs user_id, channel, thread_id for troubleshooting This maintains debugging capability for developers while protecting user privacy in production logs. Co-Authored-By: Claude Haiku 4.5 <[email protected]> --------- Co-authored-by: Claude Haiku 4.5 <[email protected]>
This commit is contained in:
co-authored by
Claude Haiku 4.5
parent
f8c56727c6
commit
34f69b31dc
@@ -738,6 +738,18 @@ impl Agent {
|
||||
}
|
||||
|
||||
async fn handle_message(&self, message: &IncomingMessage) -> Result<Option<String>, Error> {
|
||||
// Log at info level only for tracking without exposing PII (user_id can be a phone number)
|
||||
tracing::info!(message_id = %message.id, "Processing message");
|
||||
|
||||
// Log sensitive details at debug level for troubleshooting
|
||||
tracing::debug!(
|
||||
message_id = %message.id,
|
||||
user_id = %message.user_id,
|
||||
channel = %message.channel,
|
||||
thread_id = ?message.thread_id,
|
||||
"Message details"
|
||||
);
|
||||
|
||||
// Set message tool context for this turn (current channel and target)
|
||||
// For Signal, use signal_target from metadata (group:ID or phone number),
|
||||
// otherwise fall back to user_id
|
||||
@@ -786,10 +798,19 @@ impl Agent {
|
||||
|
||||
// Hydrate thread from DB if it's a historical thread not in memory
|
||||
if let Some(ref external_thread_id) = message.thread_id {
|
||||
tracing::debug!(
|
||||
message_id = %message.id,
|
||||
thread_id = %external_thread_id,
|
||||
"Hydrating thread from DB"
|
||||
);
|
||||
self.maybe_hydrate_thread(message, external_thread_id).await;
|
||||
}
|
||||
|
||||
// Resolve session and thread
|
||||
tracing::debug!(
|
||||
message_id = %message.id,
|
||||
"Resolving session and thread"
|
||||
);
|
||||
let (session, thread_id) = self
|
||||
.session_manager
|
||||
.resolve_thread(
|
||||
@@ -798,6 +819,11 @@ impl Agent {
|
||||
message.thread_id.as_deref(),
|
||||
)
|
||||
.await;
|
||||
tracing::info!(
|
||||
message_id = %message.id,
|
||||
thread_id = %thread_id,
|
||||
"Resolved session and thread"
|
||||
);
|
||||
|
||||
// Auth mode interception: if the thread is awaiting a token, route
|
||||
// the message directly to the credential store. Nothing touches
|
||||
|
||||
@@ -113,6 +113,13 @@ impl Agent {
|
||||
thread_id: Uuid,
|
||||
content: &str,
|
||||
) -> Result<SubmissionResult, Error> {
|
||||
tracing::debug!(
|
||||
message_id = %message.id,
|
||||
thread_id = %thread_id,
|
||||
content_len = content.len(),
|
||||
"Processing user input"
|
||||
);
|
||||
|
||||
// First check thread state without holding lock during I/O
|
||||
let thread_state = {
|
||||
let sess = session.lock().await;
|
||||
@@ -123,19 +130,41 @@ impl Agent {
|
||||
thread.state
|
||||
};
|
||||
|
||||
tracing::debug!(
|
||||
message_id = %message.id,
|
||||
thread_id = %thread_id,
|
||||
thread_state = ?thread_state,
|
||||
"Checked thread state"
|
||||
);
|
||||
|
||||
// Check thread state
|
||||
match thread_state {
|
||||
ThreadState::Processing => {
|
||||
tracing::warn!(
|
||||
message_id = %message.id,
|
||||
thread_id = %thread_id,
|
||||
"Thread is processing, rejecting new input"
|
||||
);
|
||||
return Ok(SubmissionResult::error(
|
||||
"Turn in progress. Use /interrupt to cancel.",
|
||||
));
|
||||
}
|
||||
ThreadState::AwaitingApproval => {
|
||||
tracing::warn!(
|
||||
message_id = %message.id,
|
||||
thread_id = %thread_id,
|
||||
"Thread awaiting approval, rejecting new input"
|
||||
);
|
||||
return Ok(SubmissionResult::error(
|
||||
"Waiting for approval. Use /interrupt to cancel.",
|
||||
));
|
||||
}
|
||||
ThreadState::Completed => {
|
||||
tracing::warn!(
|
||||
message_id = %message.id,
|
||||
thread_id = %thread_id,
|
||||
"Thread completed, rejecting new input"
|
||||
);
|
||||
return Ok(SubmissionResult::error(
|
||||
"Thread completed. Use /thread new.",
|
||||
));
|
||||
@@ -269,9 +298,20 @@ impl Agent {
|
||||
};
|
||||
|
||||
// Persist user message to DB immediately so it survives crashes
|
||||
tracing::debug!(
|
||||
message_id = %message.id,
|
||||
thread_id = %thread_id,
|
||||
"Persisting user message to DB"
|
||||
);
|
||||
self.persist_user_message(thread_id, &message.user_id, effective_content)
|
||||
.await;
|
||||
|
||||
tracing::debug!(
|
||||
message_id = %message.id,
|
||||
thread_id = %thread_id,
|
||||
"User message persisted, starting agentic loop"
|
||||
);
|
||||
|
||||
// Send thinking status
|
||||
let _ = self
|
||||
.channels
|
||||
|
||||
Reference in New Issue
Block a user