fix: full_job routine concurrency tracks linked job lifetime (#1372)

* fix: add FullJobWatcher to track full_job lifecycle for concurrency (#1318)

full_job routines previously bypassed max_concurrent and global concurrency
limits because execute_full_job() returned RunStatus::Ok immediately after
dispatch. This meant running_count was decremented and the routine_run row
was finalized before the actual job completed.

Introduce FullJobWatcher struct that polls store.get_job() every 5s until
the linked job reaches a non-active state, then maps the final JobState to
RunStatus. execute_full_job now creates and awaits the watcher, keeping both
the DB-level running row and the in-memory running_count elevated for the
full job duration.

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

* test: full_job concurrency regression tests (issue #1318)

Add two integration tests verifying full_job routine concurrency:

1. full_job_max_concurrent_blocks_second_fire_while_first_active:
   Inserts a Running routine_run (simulating an in-flight full_job) and
   verifies fire_manual returns MaxConcurrent error for max_concurrent=1.

2. global_concurrency_counts_live_full_job_runs:
   Elevates running_count to simulate a live full_job holding the global
   slot, verifies check_cron_triggers skips due routines, then releases
   the slot and verifies the routine fires.

Also makes running_count_for_test() unconditionally public so integration
tests (separate crate) can access it.

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

* style: fmt and clippy fixes for full_job concurrency tests

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

* fix: address PR review feedback on FullJobWatcher

- Add #[doc(hidden)] to running_count_for_test() to hide from public API
- Derive MAX_POLLS from POLL_INTERVAL to keep constants coupled
- Check job state before first sleep to finalize promptly for fast jobs
- Update execute_full_job doc comment to reflect blocking behavior

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <[email protected]>

src/agent/routine_engine.rs

@@ -88,6 +88,12 @@ impl RoutineEngine {
         }
     }
 
+    /// Expose the running count for integration tests.
+    #[doc(hidden)]
+    pub fn running_count_for_test(&self) -> &Arc<AtomicUsize> {
+        &self.running_count
+    }
+
     /// Refresh the in-memory event trigger cache from DB.
     pub async fn refresh_event_cache(&self) {
         match self.store.list_event_routines().await {
@@ -508,6 +514,88 @@ impl RoutineEngine {
     }
 }
 
+/// Watches a dispatched full_job until the linked scheduler job completes.
+///
+/// Polls `store.get_job(job_id)` at a fixed interval until the job leaves
+/// an active state (Pending/InProgress/Stuck). Maps the final `JobState` to
+/// a `RunStatus` for the routine run.
+struct FullJobWatcher {
+    store: Arc<dyn Database>,
+    job_id: Uuid,
+    routine_name: String,
+}
+
+impl FullJobWatcher {
+    /// Poll interval between DB checks.
+    const POLL_INTERVAL: Duration = Duration::from_secs(5);
+    /// Safety ceiling: 24 hours, derived from POLL_INTERVAL.
+    const MAX_POLLS: u32 = (24 * 60 * 60) / Self::POLL_INTERVAL.as_secs() as u32;
+
+    fn new(store: Arc<dyn Database>, job_id: Uuid, routine_name: String) -> Self {
+        Self {
+            store,
+            job_id,
+            routine_name,
+        }
+    }
+
+    /// Block until the linked job finishes and return the mapped status + summary.
+    async fn wait_for_completion(&self) -> (RunStatus, Option<String>) {
+        let mut polls = 0u32;
+
+        let final_status = loop {
+            // Check job state before sleeping so we finalize promptly
+            // if the job is already done (e.g. fast-failing jobs).
+            match self.store.get_job(self.job_id).await {
+                Ok(Some(job_ctx)) => {
+                    if !job_ctx.state.is_active() {
+                        break Self::map_job_state(&job_ctx.state);
+                    }
+                }
+                Ok(None) => {
+                    tracing::warn!(
+                        routine = %self.routine_name,
+                        job_id = %self.job_id,
+                        "full_job disappeared from DB while polling"
+                    );
+                    break RunStatus::Failed;
+                }
+                Err(e) => {
+                    tracing::error!(
+                        routine = %self.routine_name,
+                        job_id = %self.job_id,
+                        "Error polling full_job state: {}", e
+                    );
+                    break RunStatus::Failed;
+                }
+            }
+
+            polls += 1;
+            if polls >= Self::MAX_POLLS {
+                tracing::error!(
+                    routine = %self.routine_name,
+                    job_id = %self.job_id,
+                    "full_job timed out after 24 hours, treating as failed"
+                );
+                break RunStatus::Failed;
+            }
+
+            tokio::time::sleep(Self::POLL_INTERVAL).await;
+        };
+
+        let summary = format!("Job {} finished ({})", self.job_id, final_status);
+        (final_status, Some(summary))
+    }
+
+    fn map_job_state(state: &crate::context::JobState) -> RunStatus {
+        use crate::context::JobState;
+        match state {
+            JobState::Failed | JobState::Cancelled => RunStatus::Failed,
+            _ => RunStatus::Ok, // Completed / Submitted / Accepted
+        }
+    }
+}
+
 /// Shared context passed to the execution function.
 struct EngineContext {
     config: RoutineConfig,
@@ -682,8 +770,10 @@ fn sanitize_routine_name(name: &str) -> String {
 ///
 /// Fire-and-forget: creates a job via `Scheduler::dispatch_job` (which handles
 /// creation, metadata, persistence, and scheduling), links the routine run to
-/// the job, and returns immediately. The job runs independently via the
-/// existing Worker/Scheduler with full tool access.
+/// the job, then watches it via `FullJobWatcher` until it reaches a
+/// non-active state (not Pending/InProgress/Stuck). Returns the final
+/// `RunStatus` mapped from the job outcome. This keeps the routine run
+/// active for the full job lifetime so concurrency guardrails apply.
 async fn execute_full_job(
     ctx: &EngineContext,
     routine: &Routine,
@@ -738,13 +828,15 @@ async fn execute_full_job(
         routine = %routine.name,
         job_id = %job_id,
         max_iterations = max_iterations,
-        "Dispatched full job for routine"
+        "Dispatched full job for routine, watching for completion"
     );
 
-    let summary = format!(
-        "Dispatched job {job_id} for full execution with tool access (max_iterations: {max_iterations})"
-    );
-    Ok((RunStatus::Ok, Some(summary), None))
+    // Watch the job until it finishes — keeps the routine run active
+    // so concurrency guardrails (running_count, routine_runs status)
+    // remain enforced for the full job lifetime.
+    let watcher = FullJobWatcher::new(ctx.store.clone(), job_id, routine.name.clone());
+    let (status, summary) = watcher.wait_for_completion().await;
+    Ok((status, summary, None))
 }
 
 /// Execute a lightweight routine with optional tool support.