fix(tests): eliminate env mutex poison cascade (#1558)

* fix(tests): eliminate env mutex poison cascade and fix test flakiness

The shared ENV_MUTEX used by ~68 config tests would cascade a single
test panic into failures across every module. Replace all .unwrap() /
.expect() lock acquisitions with a poison-recovering lock_env() helper.
Consolidate rogue module-local ENV_LOCK instances (workspace, orchestrator,
bootstrap) onto the shared global mutex to prevent cross-module races.

Also fixes:
- gateway user_id fallback was hardcoded to "default" instead of owner_id
- test_ironclaw_env_path used LazyLock which is order-dependent

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

* test(helpers): add regression test for lock_env poison recovery

Satisfies the regression-test-check CI gate by adding a test that
intentionally poisons ENV_MUTEX and verifies lock_env() recovers.

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

* fix(ci): detect test changes inside #[cfg(test)] regions

The regression test check relied on git diff -W to expand context to
function boundaries, but git doesn't recognize Rust `mod tests {}` as a
function boundary. Changes to imports, helpers, or lock calls inside
test modules were invisible to the check.

Add a line-level fallback: for each changed .rs file, find where
#[cfg(test)] starts and check if any diff hunk targets a line at or
after that boundary. This catches edits anywhere inside test modules
regardless of git's language awareness.

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

* fix: address PR review feedback

- Clear ENV_MUTEX poison after regression test so it doesn't leave
  global state dirty for subsequent tests.
- Fix CI regression-test-check to match #[cfg(test)] only when followed
  by `mod` (the test module pattern), avoiding false positives from
  standalone #[cfg(test)] items like statics or functions.

Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <[email protected]>
This commit is contained in:
Illia Polosukhin
2026-03-22 14:36:24 -07:00
committed by GitHub
co-authored by Claude Opus 4.6
parent fbce9a5fe3
commit 3aa36c8f55
19 changed files with 192 additions and 143 deletions
+4 -4
View File
@@ -692,7 +692,7 @@ mod tests {
}
}
let _mutex = crate::config::helpers::ENV_MUTEX.lock().expect("env mutex");
let _mutex = crate::config::helpers::lock_env();
let prev = std::env::var("LLM_BACKEND").ok();
// SAFETY: Under ENV_MUTEX, no concurrent env access.
unsafe {
@@ -812,7 +812,7 @@ mod tests {
#[test]
fn check_llm_config_shows_nearai_model_for_nearai_backend() {
let _guard = crate::config::helpers::ENV_MUTEX.lock().expect("env mutex");
let _guard = crate::config::helpers::lock_env();
// SAFETY: Under ENV_MUTEX, no concurrent env access.
unsafe {
std::env::remove_var("LLM_BACKEND");
@@ -839,7 +839,7 @@ mod tests {
#[test]
fn check_embeddings_disabled_by_default_returns_skip() {
let _guard = crate::config::helpers::ENV_MUTEX.lock().expect("env mutex");
let _guard = crate::config::helpers::lock_env();
// SAFETY: Under ENV_MUTEX.
unsafe {
std::env::remove_var("EMBEDDING_ENABLED");
@@ -861,7 +861,7 @@ mod tests {
#[test]
fn check_routines_enabled_by_default() {
let _guard = crate::config::helpers::ENV_MUTEX.lock().expect("env mutex");
let _guard = crate::config::helpers::lock_env();
// SAFETY: Under ENV_MUTEX.
unsafe {
std::env::remove_var("ROUTINES_ENABLED");
+12 -12
View File
@@ -758,7 +758,7 @@ mod tests {
use crate::cli::oauth_defaults::{
builtin_credentials, callback_host, callback_url, is_loopback_host, landing_html,
};
use crate::config::helpers::ENV_MUTEX;
use crate::config::helpers::lock_env;
#[test]
fn test_is_loopback_host() {
@@ -775,7 +775,7 @@ mod tests {
#[test]
fn test_callback_host_default() {
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
let _guard = lock_env();
let original = std::env::var("OAUTH_CALLBACK_HOST").ok();
// SAFETY: Under ENV_MUTEX, no concurrent env access.
unsafe {
@@ -792,7 +792,7 @@ mod tests {
#[test]
fn test_callback_host_env_override() {
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
let _guard = lock_env();
let original_host = std::env::var("OAUTH_CALLBACK_HOST").ok();
let original_url = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
// SAFETY: Under ENV_MUTEX, no concurrent env access.
@@ -819,7 +819,7 @@ mod tests {
#[test]
fn test_callback_url_default() {
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
let _guard = lock_env();
// Clear both env vars to test default behavior
let original_url = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
let original_host = std::env::var("OAUTH_CALLBACK_HOST").ok();
@@ -843,7 +843,7 @@ mod tests {
#[test]
fn test_callback_url_env_override() {
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
let _guard = lock_env();
let original = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
// SAFETY: Under ENV_MUTEX, no concurrent env access.
unsafe {
@@ -1008,7 +1008,7 @@ mod tests {
#[test]
fn test_use_gateway_callback_false_by_default() {
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
let _guard = lock_env();
let original = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
// SAFETY: Under ENV_MUTEX, no concurrent env access.
unsafe {
@@ -1024,7 +1024,7 @@ mod tests {
#[test]
fn test_use_gateway_callback_true_for_hosted() {
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
let _guard = lock_env();
let original = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
// SAFETY: Under ENV_MUTEX, no concurrent env access.
unsafe {
@@ -1045,7 +1045,7 @@ mod tests {
#[test]
fn test_use_gateway_callback_false_for_localhost() {
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
let _guard = lock_env();
let original = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
// SAFETY: Under ENV_MUTEX, no concurrent env access.
unsafe {
@@ -1063,7 +1063,7 @@ mod tests {
#[test]
fn test_use_gateway_callback_false_for_empty() {
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
let _guard = lock_env();
let original = std::env::var("IRONCLAW_OAUTH_CALLBACK_URL").ok();
// SAFETY: Under ENV_MUTEX, no concurrent env access.
unsafe {
@@ -1083,7 +1083,7 @@ mod tests {
fn test_build_platform_state_with_instance() {
use crate::cli::oauth_defaults::{build_platform_state, decode_hosted_oauth_state};
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
let _guard = lock_env();
let original = std::env::var("IRONCLAW_INSTANCE_NAME").ok();
// SAFETY: Under ENV_MUTEX, no concurrent env access.
unsafe {
@@ -1107,7 +1107,7 @@ mod tests {
fn test_build_platform_state_without_instance() {
use crate::cli::oauth_defaults::{build_platform_state, decode_hosted_oauth_state};
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
let _guard = lock_env();
let original = std::env::var("IRONCLAW_INSTANCE_NAME").ok();
let original_oc = std::env::var("OPENCLAW_INSTANCE_NAME").ok();
// SAFETY: Under ENV_MUTEX, no concurrent env access.
@@ -1134,7 +1134,7 @@ mod tests {
fn test_build_platform_state_with_openclaw_instance() {
use crate::cli::oauth_defaults::{build_platform_state, decode_hosted_oauth_state};
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
let _guard = lock_env();
let original_ic = std::env::var("IRONCLAW_INSTANCE_NAME").ok();
let original_oc = std::env::var("OPENCLAW_INSTANCE_NAME").ok();
// SAFETY: Under ENV_MUTEX, no concurrent env access.