Merge remote-tracking branch 'origin/staging' into refactor/architectural-hardening

# Conflicts:
#	src/agent/routine.rs

scripts/check_no_panics.py

@@ -0,0 +1,360 @@
+#!/usr/bin/env python3
+# Requires Python 3.10+ for PEP 604 union syntax such as `int | None`.
+
+import argparse
+import pathlib
+import re
+import subprocess
+import sys
+import unittest
+from dataclasses import dataclass
+
+
+PANIC_PATTERN = re.compile(r"\.(?:unwrap|expect)\(|(?<!_)assert(?:_eq|_ne)?!")
+TEST_ATTR_PATTERN = re.compile(
+    r"^\s*#\s*\[\s*(?:"
+    r"test"
+    r"|tokio::test(?:\s*\([^]]*\))?"
+    r"|rstest(?:\s*\([^]]*\))?"
+    r"|test_case(?:\s*\([^]]*\))?"
+    r"|cfg\s*\([^]]*\btest\b[^]]*\)"
+    r")\s*\]"
+)
+ITEM_PATTERN = re.compile(
+    r"^\s*"
+    r"(?:(?:pub(?:\([^)]*\))?|crate)\s+)?"
+    r"(?:(?:async|unsafe|const)\s+)*"
+    r"(fn|mod|struct|enum|trait|union|impl)\b"
+    r"(?:\s+([A-Za-z_][A-Za-z0-9_]*))?"
+)
+
+
+@dataclass
+class LexerState:
+    block_comment_depth: int = 0
+    in_string: bool = False
+    string_escape: bool = False
+    in_char: bool = False
+    char_escape: bool = False
+    raw_string_hashes: int | None = None
+
+
+def run_git(*args: str) -> str:
+    result = subprocess.run(
+        ["git", *args],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return result.stdout
+
+
+def sanitize_line(line: str, state: LexerState) -> str:
+    chars = list(line)
+    out = [" "] * len(chars)
+    i = 0
+
+    while i < len(chars):
+        ch = chars[i]
+        nxt = chars[i + 1] if i + 1 < len(chars) else ""
+
+        if state.block_comment_depth:
+            if ch == "/" and nxt == "*":
+                state.block_comment_depth += 1
+                i += 2
+                continue
+            if ch == "*" and nxt == "/":
+                state.block_comment_depth -= 1
+                i += 2
+                continue
+            i += 1
+            continue
+
+        if state.raw_string_hashes is not None:
+            if ch == '"':
+                hashes = 0
+                j = i + 1
+                while j < len(chars) and chars[j] == "#":
+                    hashes += 1
+                    j += 1
+                if hashes == state.raw_string_hashes:
+                    state.raw_string_hashes = None
+                    i = j
+                    continue
+            i += 1
+            continue
+
+        if state.in_string:
+            if state.string_escape:
+                state.string_escape = False
+            elif ch == "\\":
+                state.string_escape = True
+            elif ch == '"':
+                state.in_string = False
+            i += 1
+            continue
+
+        if state.in_char:
+            if state.char_escape:
+                state.char_escape = False
+            elif ch == "\\":
+                state.char_escape = True
+            elif ch == "'":
+                state.in_char = False
+            i += 1
+            continue
+
+        if ch == "/" and nxt == "/":
+            break
+        if ch == "/" and nxt == "*":
+            state.block_comment_depth += 1
+            i += 2
+            continue
+        if ch == "r":
+            j = i + 1
+            while j < len(chars) and chars[j] == "#":
+                j += 1
+            if j < len(chars) and chars[j] == '"':
+                state.raw_string_hashes = j - i - 1
+                i = j + 1
+                continue
+        if ch == '"':
+            state.in_string = True
+            i += 1
+            continue
+        if ch == "'":
+            # This can misclassify lifetimes like `'a` as char literals. That only
+            # risks false negatives by masking later code on the same line.
+            state.in_char = True
+            i += 1
+            continue
+
+        out[i] = ch
+        i += 1
+
+    return "".join(out)
+
+
+def is_test_item(line: str, pending_test_attr: bool) -> tuple[bool, bool]:
+    match = ITEM_PATTERN.match(line)
+    if not match:
+        return False, False
+
+    kind, name = match.groups()
+    named_tests_module = kind == "mod" and name == "tests"
+    return True, pending_test_attr or named_tests_module
+
+
+def line_test_contexts(lines: list[str]) -> list[bool]:
+    contexts = [False] * len(lines)
+    lexer = LexerState()
+    block_stack: list[bool] = []
+    pending_test_attr = False
+    pending_block_context: bool | None = None
+
+    for idx, raw in enumerate(lines):
+        code = sanitize_line(raw, lexer)
+        stripped = code.strip()
+        current_context = block_stack[-1] if block_stack else False
+
+        if TEST_ATTR_PATTERN.match(stripped):
+            pending_test_attr = True
+
+        item_found, item_is_test = is_test_item(code, pending_test_attr)
+        if item_found:
+            pending_block_context = item_is_test or current_context
+            pending_test_attr = False
+        elif stripped and not stripped.startswith("#[") and pending_test_attr:
+            pending_test_attr = False
+
+        contexts[idx] = current_context or bool(pending_block_context)
+
+        for ch in code:
+            if ch == "{":
+                if pending_block_context is not None:
+                    block_stack.append(pending_block_context)
+                    pending_block_context = None
+                else:
+                    block_stack.append(block_stack[-1] if block_stack else False)
+            elif ch == "}" and block_stack:
+                block_stack.pop()
+
+        if stripped.endswith(";"):
+            pending_block_context = None
+
+    return contexts
+
+
+def changed_rust_files(base: str, head: str) -> list[pathlib.Path]:
+    output = run_git("diff", "--name-only", f"{base}...{head}", "--", "src", "crates")
+    files = []
+    for line in output.splitlines():
+        if line.endswith(".rs") and (line.startswith("src/") or line.startswith("crates/")):
+            files.append(pathlib.Path(line))
+    return files
+
+
+def added_lines_for_file(base: str, head: str, path: pathlib.Path) -> set[int]:
+    diff = run_git("diff", "--unified=0", f"{base}...{head}", "--", str(path))
+    added: set[int] = set()
+    current_line = 0
+
+    for line in diff.splitlines():
+        if line.startswith("@@"):
+            match = re.search(r"\+(\d+)(?:,(\d+))?", line)
+            if not match:
+                continue
+            current_line = int(match.group(1))
+            continue
+        if line.startswith("+++ ") or line.startswith("--- "):
+            continue
+        if line.startswith("+"):
+            added.add(current_line)
+            current_line += 1
+        elif line.startswith("-"):
+            continue
+        else:
+            current_line += 1
+
+    return added
+
+
+def collect_violations(base: str, head: str) -> list[tuple[str, int, str]]:
+    violations: list[tuple[str, int, str]] = []
+
+    for path in changed_rust_files(base, head):
+        if not path.exists():
+            continue
+        added_lines = added_lines_for_file(base, head, path)
+        if not added_lines:
+            continue
+
+        lines = path.read_text(encoding="utf-8").splitlines()
+        contexts = line_test_contexts(lines)
+        lexer = LexerState()
+        sanitized = [sanitize_line(line, lexer) for line in lines]
+
+        for line_no in sorted(added_lines):
+            if line_no < 1 or line_no > len(lines):
+                continue
+            if contexts[line_no - 1]:
+                continue
+            if "// safety:" in lines[line_no - 1]:
+                continue
+            if PANIC_PATTERN.search(sanitized[line_no - 1]):
+                violations.append((str(path), line_no, lines[line_no - 1].rstrip()))
+
+    return violations
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--base", required=False, default="origin/staging")
+    parser.add_argument("--head", required=False, default="HEAD")
+    parser.add_argument("--self-test", action="store_true")
+    args = parser.parse_args()
+
+    if args.self_test:
+        suite = unittest.defaultTestLoader.loadTestsFromTestCase(CheckNoPanicsTests)
+        result = unittest.TextTestRunner(verbosity=2).run(suite)
+        return 0 if result.wasSuccessful() else 1
+
+    violations = collect_violations(args.base, args.head)
+    if not violations:
+        print("OK: No panic-inducing calls in changed production code.")
+        return 0
+
+    print("::error::Found panic-style calls outside test-only Rust code.")
+    print("Production code must use proper error handling instead of panicking.")
+    print("Suppress false positives with an inline '// safety: <reason>' comment.")
+    print("")
+    for path, line_no, line in violations[:20]:
+        print(f"{path}:{line_no}: {line}")
+    print("")
+    print(f"Total: {len(violations)} violation(s)")
+    return 1
+
+
+class CheckNoPanicsTests(unittest.TestCase):
+    def test_cfg_test_module_marks_inner_lines(self) -> None:
+        lines = [
+            "#[cfg(test)]\n",
+            "mod tests {\n",
+            "    assert!(true);\n",
+            "}\n",
+            "fn prod() {\n",
+            "    value.expect(\"boom\");\n",
+            "}\n",
+        ]
+
+        contexts = line_test_contexts(lines)
+
+        self.assertTrue(contexts[1])
+        self.assertTrue(contexts[2])
+        self.assertFalse(contexts[4])
+        self.assertFalse(contexts[5])
+
+    def test_test_function_marks_body_only(self) -> None:
+        lines = [
+            "#[test]\n",
+            "fn it_works(\n",
+            ") {\n",
+            "    assert_eq!(2 + 2, 4);\n",
+            "}\n",
+            "fn prod() {\n",
+            "    assert!(ready);\n",
+            "}\n",
+        ]
+
+        contexts = line_test_contexts(lines)
+
+        self.assertTrue(contexts[1])
+        self.assertTrue(contexts[2])
+        self.assertTrue(contexts[3])
+        self.assertFalse(contexts[5])
+        self.assertFalse(contexts[6])
+
+    def test_proc_macro_test_attrs_mark_body_only(self) -> None:
+        attrs = [
+            "tokio::test",
+            'tokio::test(flavor = "multi_thread", worker_threads = 4)',
+            "rstest",
+            "test_case(1, 2)",
+            "cfg(all(test, unix))",
+        ]
+
+        for attr in attrs:
+            with self.subTest(attr=attr):
+                lines = [
+                    f"#[{attr}]\n",
+                    "fn it_works() {\n",
+                    '    value.expect("allowed in test");\n',
+                    "}\n",
+                    "fn prod() {\n",
+                    '    value.expect("boom");\n',
+                    "}\n",
+                ]
+
+                contexts = line_test_contexts(lines)
+
+                self.assertTrue(contexts[1])
+                self.assertTrue(contexts[2])
+                self.assertFalse(contexts[4])
+                self.assertFalse(contexts[5])
+
+    def test_named_tests_module_marks_context(self) -> None:
+        lines = [
+            "mod tests {\n",
+            "    fn helper() {\n",
+            "        assert!(true);\n",
+            "    }\n",
+            "}\n",
+        ]
+
+        contexts = line_test_contexts(lines)
+
+        self.assertTrue(all(contexts))
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/ci/delta_lint.sh

@@ -0,0 +1,216 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# Delta lint: only fail on clippy warnings/errors that touch changed lines.
+# Compares the current branch against the merge base with the upstream default branch.
+
+CLIPPY_OUT=""
+DIFF_OUT=""
+CLIPPY_STDERR=""
+
+cleanup() {
+    [ -n "$CLIPPY_OUT" ] && rm -f "$CLIPPY_OUT"
+    [ -n "$DIFF_OUT" ] && rm -f "$DIFF_OUT"
+    [ -n "$CLIPPY_STDERR" ] && rm -f "$CLIPPY_STDERR"
+}
+trap cleanup EXIT
+
+# Verify python3 is available (needed for diagnostic filtering)
+if ! command -v python3 &>/dev/null; then
+    echo "ERROR: python3 is required for delta lint but not found"
+    exit 1
+fi
+
+# Accept optional remote name argument; default to dynamic detection
+REMOTE="${1:-}"
+
+# Determine the upstream base ref dynamically
+BASE_REF=""
+if [ -n "$REMOTE" ]; then
+    # Use the provided remote name
+    if [ -z "$BASE_REF" ]; then
+        BASE_REF=$(git symbolic-ref "refs/remotes/$REMOTE/HEAD" 2>/dev/null | sed 's|refs/remotes/||' || true)
+    fi
+    if [ -z "$BASE_REF" ] && git rev-parse --verify "$REMOTE/main" &>/dev/null; then
+        BASE_REF="$REMOTE/main"
+    fi
+    if [ -z "$BASE_REF" ] && git rev-parse --verify "$REMOTE/master" &>/dev/null; then
+        BASE_REF="$REMOTE/master"
+    fi
+else
+    # Try the remote HEAD symbolic ref (works for any default branch name)
+    if [ -z "$BASE_REF" ]; then
+        BASE_REF=$(git symbolic-ref refs/remotes/origin/HEAD 2>/dev/null | sed 's|refs/remotes/||' || true)
+    fi
+    # Fall back to common default branch names
+    if [ -z "$BASE_REF" ] && git rev-parse --verify origin/main &>/dev/null; then
+        BASE_REF="origin/main"
+    fi
+    if [ -z "$BASE_REF" ] && git rev-parse --verify origin/master &>/dev/null; then
+        BASE_REF="origin/master"
+    fi
+fi
+if [ -z "$BASE_REF" ]; then
+    echo "WARNING: could not determine upstream base branch, skipping delta lint"
+    exit 0
+fi
+
+# Compute merge base
+BASE=$(git merge-base "$BASE_REF" HEAD 2>/dev/null) || {
+    echo "WARNING: git merge-base failed for $BASE_REF, skipping delta lint"
+    exit 0
+}
+
+# Find changed .rs files
+CHANGED_RS=$(git diff --name-only "$BASE" -- '*.rs' || true)
+if [ -z "$CHANGED_RS" ]; then
+    echo "==> delta lint: no .rs files changed, skipping"
+    exit 0
+fi
+
+echo "==> delta lint: checking changed lines since $(echo "$BASE" | head -c 10)..."
+
+# Extract unified-0 diff for changed line ranges
+DIFF_OUT=$(mktemp "${TMPDIR:-/tmp}/ironclaw-diff.XXXXXX")
+git diff --unified=0 "$BASE" -- '*.rs' > "$DIFF_OUT"
+
+# Run clippy with JSON output (stderr shows compilation progress/errors)
+CLIPPY_OUT=$(mktemp "${TMPDIR:-/tmp}/ironclaw-clippy.XXXXXX")
+CLIPPY_STDERR=$(mktemp "${TMPDIR:-/tmp}/ironclaw-clippy-err.XXXXXX")
+cargo clippy --locked --all-targets --message-format=json > "$CLIPPY_OUT" 2>"$CLIPPY_STDERR" || true
+
+# Show compilation errors if clippy produced no JSON output
+if [ ! -s "$CLIPPY_OUT" ] && [ -s "$CLIPPY_STDERR" ]; then
+    echo "ERROR: clippy failed to produce output. Compilation errors:"
+    cat "$CLIPPY_STDERR"
+    exit 1
+fi
+
+# Get repo root for path normalization in Python
+REPO_ROOT="$(git rev-parse --show-toplevel)"
+
+# Filter clippy diagnostics against changed line ranges
+python3 - "$DIFF_OUT" "$CLIPPY_OUT" "$REPO_ROOT" <<'PYEOF'
+import json
+import re
+import sys
+import os
+
+def parse_diff(diff_path):
+    """Parse unified-0 diff to extract {file: [[start, end], ...]} changed ranges."""
+    changed = {}
+    current_file = None
+    with open(diff_path) as f:
+        for line in f:
+            # Match +++ b/path/to/file.rs or +++ /dev/null (deletion)
+            if line.startswith('+++ /dev/null'):
+                current_file = None
+                continue
+            m = re.match(r'^\+\+\+ b/(.+)$', line)
+            if m:
+                current_file = m.group(1)
+                if current_file not in changed:
+                    changed[current_file] = []
+                continue
+            # Match @@ hunk headers: @@ -old,count +new,count @@
+            m = re.match(r'^@@ .+ \+(\d+)(?:,(\d+))? @@', line)
+            if m and current_file:
+                start = int(m.group(1))
+                count = int(m.group(2)) if m.group(2) is not None else 1
+                if count == 0:
+                    continue
+                end = start + count - 1
+                changed[current_file].append([start, end])
+    return changed
+
+def normalize_path(path, repo_root):
+    """Normalize absolute path to relative (from repo root)."""
+    if os.path.isabs(path):
+        if path.startswith(repo_root):
+            return os.path.relpath(path, repo_root)
+    return path
+
+def in_changed_range(file_path, line_start, line_end, changed_ranges, repo_root):
+    """Check if file:[line_start, line_end] overlaps any changed range."""
+    rel = normalize_path(file_path, repo_root)
+    ranges = changed_ranges.get(rel)
+    if not ranges:
+        return False
+    return any(start <= line_end and line_start <= end for start, end in ranges)
+
+def main():
+    diff_path = sys.argv[1]
+    clippy_path = sys.argv[2]
+    repo_root = sys.argv[3]
+
+    changed_ranges = parse_diff(diff_path)
+
+    blocking = []
+    baseline = []
+
+    with open(clippy_path) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                msg = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+            if msg.get("reason") != "compiler-message":
+                continue
+
+            cm = msg.get("message", {})
+            level = cm.get("level", "")
+            if level not in ("warning", "error"):
+                continue
+
+            rendered = cm.get("rendered", "").strip()
+
+            # Errors are always blocking regardless of location
+            if level == "error":
+                blocking.append(rendered)
+                continue
+
+            # For warnings, only block if they overlap changed lines
+            spans = cm.get("spans", [])
+            primary = None
+            for s in spans:
+                if s.get("is_primary"):
+                    primary = s
+                    break
+            if not primary:
+                if spans:
+                    primary = spans[0]
+                else:
+                    baseline.append(rendered)
+                    continue
+
+            file_name = primary.get("file_name", "")
+            line_start = primary.get("line_start", 0)
+            line_end = primary.get("line_end", line_start)
+
+            if in_changed_range(file_name, line_start, line_end, changed_ranges, repo_root):
+                blocking.append(rendered)
+            else:
+                baseline.append(rendered)
+
+    if baseline:
+        print(f"\n--- Baseline warnings (not in changed lines, informational) [{len(baseline)}] ---")
+        for w in baseline[:10]:
+            print(w)
+        if len(baseline) > 10:
+            print(f"  ... and {len(baseline) - 10} more")
+
+    if blocking:
+        print(f"\n*** BLOCKING: {len(blocking)} issue(s) in changed lines ***")
+        for w in blocking:
+            print(w)
+        sys.exit(1)
+    else:
+        print("\n==> delta lint: passed (no issues in changed lines)")
+        sys.exit(0)
+
+if __name__ == "__main__":
+    main()
+PYEOF

scripts/ci/quality_gate.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "==> fmt check"
+cargo fmt --all -- --check
+
+echo "==> clippy (correctness)"
+cargo clippy --locked --all-targets -- -D clippy::correctness
+
+if [ "${IRONCLAW_PREPUSH_TEST:-1}" = "1" ]; then
+    echo "==> tests (skip with IRONCLAW_PREPUSH_TEST=0)"
+    cargo test --locked --lib
+fi

scripts/dev-setup.sh

@@ -56,6 +56,9 @@ if [ -n "$HOOKS_DIR" ]; then
     echo "  commit-msg hook installed (regression test enforcement)"
     ln -sf "$SCRIPTS_ABS/pre-commit-safety.sh" "$HOOKS_DIR/pre-commit"
     echo "  pre-commit hook installed (UTF-8, case-sensitivity, /tmp, redaction checks)"
+    REPO_ROOT="$(git rev-parse --show-toplevel)"
+    ln -sf "$REPO_ROOT/.githooks/pre-push" "$HOOKS_DIR/pre-push"
+    echo "  pre-push hook installed (quality gate + optional delta lint)"
 else
     echo "  Skipped: not a git repository"
 fi

scripts/pre-commit-safety.sh

@@ -134,8 +134,19 @@ fi
 #    Excludes test files, test modules, and debug_assert (compiled out in release).
 #    Suppress with "// safety: <reason>".
 PROD_DIFF="$DIFF_OUTPUT"
-# Strip hunks from test-only files (tests/ directory, *_test.rs, test_*.rs)
-PROD_DIFF=$(echo "$PROD_DIFF" | grep -v '^+++ b/tests/' || true)
+# Strip all hunks from test-only files (tests/ directory, *_test.rs, test_*.rs, benches/)
+PROD_DIFF=$(echo "$PROD_DIFF" | awk '
+    /^diff --git/ { in_test_file = ($0 ~ /tests\/|_test\.rs|test_.*\.rs|benches\//) }
+    !in_test_file { print }
+' || true)
+# Strip hunks whose @@ context line indicates a test module.
+# git diff includes the enclosing function/module name after @@.
+# Only match `mod tests` (the conventional #[cfg(test)] module) — do NOT
+# match `fn test_*` because production code can have functions named test_*.
+PROD_DIFF=$(echo "$PROD_DIFF" | awk '
+    /^@@ / { in_test = ($0 ~ /mod tests/) }
+    !in_test { print }
+' || true)
 if echo "$PROD_DIFF" | grep -nE '^\+' \
     | grep -E '\.(unwrap|expect)\(|[^_]assert(_eq|_ne)?!' \
     | grep -vE 'debug_assert|// safety:|#\[cfg\(test\)\]|#\[test\]|mod tests' \