mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-25 14:53:34 +00:00
* ci: enhance coverage workflow with feature matrix, postgres, and E2E Replace single-config coverage job with a multi-job pipeline: - Mirror test.yml's 3-config feature matrix (all-features, default, libsql-only) - Add PostgreSQL service (pgvector/pgvector:pg16) with migrations for postgres configs so integration tests actually run instead of skipping - Add E2E coverage job using cargo-llvm-cov instrumented binary with Playwright browser tests - Add coverage-gate roll-up job for branch protection - Upload per-config flags to Codecov (all-features, default, libsql-only, e2e) - Forward LLVM coverage env vars in E2E conftest.py so profraw data lands where cargo-llvm-cov report expects it [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: address PR review feedback on coverage workflow - Avoid setting DATABASE_URL to empty string for libsql-only config; use $GITHUB_ENV conditional step so the var is unset entirely - Add set -euo pipefail and psql -v ON_ERROR_STOP=1 to migrations so SQL errors fail the job immediately [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> --------- Co-authored-by: Claude Opus 4.6 <[email protected]>
169 lines
5.8 KiB
Python
169 lines
5.8 KiB
Python
"""pytest fixtures for E2E tests.
|
|
|
|
Session-scoped: build binary, start mock LLM, start ironclaw, launch browser.
|
|
Function-scoped: fresh browser context and page per test.
|
|
"""
|
|
|
|
import asyncio
|
|
import os
|
|
import signal
|
|
import socket
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from helpers import AUTH_TOKEN, wait_for_port_line, wait_for_ready
|
|
|
|
# Project root (two levels up from tests/e2e/)
|
|
ROOT = Path(__file__).resolve().parent.parent.parent
|
|
|
|
# Temp directory for the libSQL database file (cleaned up automatically)
|
|
_DB_TMPDIR = tempfile.TemporaryDirectory(prefix="ironclaw-e2e-")
|
|
|
|
|
|
def _find_free_port() -> int:
|
|
"""Bind to port 0 and return the OS-assigned port."""
|
|
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
|
s.bind(("127.0.0.1", 0))
|
|
return s.getsockname()[1]
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def ironclaw_binary():
|
|
"""Ensure ironclaw binary is built. Returns the binary path."""
|
|
binary = ROOT / "target" / "debug" / "ironclaw"
|
|
if not binary.exists():
|
|
print("Building ironclaw (this may take a while)...")
|
|
subprocess.run(
|
|
["cargo", "build", "--no-default-features", "--features", "libsql"],
|
|
cwd=ROOT,
|
|
check=True,
|
|
timeout=600,
|
|
)
|
|
assert binary.exists(), f"Binary not found at {binary}"
|
|
return str(binary)
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
async def mock_llm_server():
|
|
"""Start the mock LLM server. Yields the base URL."""
|
|
server_script = Path(__file__).parent / "mock_llm.py"
|
|
proc = await asyncio.create_subprocess_exec(
|
|
sys.executable, str(server_script), "--port", "0",
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
try:
|
|
port = await wait_for_port_line(proc, r"MOCK_LLM_PORT=(\d+)", timeout=10)
|
|
url = f"http://127.0.0.1:{port}"
|
|
await wait_for_ready(f"{url}/v1/models", timeout=10)
|
|
yield url
|
|
finally:
|
|
proc.send_signal(signal.SIGTERM)
|
|
try:
|
|
await asyncio.wait_for(proc.wait(), timeout=5)
|
|
except asyncio.TimeoutError:
|
|
proc.kill()
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
async def ironclaw_server(ironclaw_binary, mock_llm_server):
|
|
"""Start the ironclaw gateway. Yields the base URL."""
|
|
gateway_port = _find_free_port()
|
|
env = {
|
|
# Minimal env: PATH for process spawning, HOME for Rust/cargo defaults
|
|
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
|
|
"HOME": os.environ.get("HOME", "/tmp"),
|
|
"RUST_LOG": "ironclaw=info",
|
|
"RUST_BACKTRACE": "1",
|
|
"GATEWAY_ENABLED": "true",
|
|
"GATEWAY_HOST": "127.0.0.1",
|
|
"GATEWAY_PORT": str(gateway_port),
|
|
"GATEWAY_AUTH_TOKEN": AUTH_TOKEN,
|
|
"GATEWAY_USER_ID": "e2e-tester",
|
|
"CLI_ENABLED": "false",
|
|
"LLM_BACKEND": "openai_compatible",
|
|
"LLM_BASE_URL": mock_llm_server,
|
|
"LLM_MODEL": "mock-model",
|
|
"DATABASE_BACKEND": "libsql",
|
|
"LIBSQL_PATH": os.path.join(_DB_TMPDIR.name, "e2e.db"),
|
|
"SANDBOX_ENABLED": "false",
|
|
"SKILLS_ENABLED": "true",
|
|
"ROUTINES_ENABLED": "false",
|
|
"HEARTBEAT_ENABLED": "false",
|
|
"EMBEDDING_ENABLED": "false",
|
|
# Prevent onboarding wizard from triggering
|
|
"ONBOARD_COMPLETED": "true",
|
|
}
|
|
# Forward LLVM coverage instrumentation env vars when present
|
|
# (allows cargo-llvm-cov to collect profraw data from E2E runs)
|
|
for key in ("LLVM_PROFILE_FILE", "CARGO_LLVM_COV", "CARGO_LLVM_COV_SHOW_ENV",
|
|
"CARGO_LLVM_COV_TARGET_DIR"):
|
|
val = os.environ.get(key)
|
|
if val is not None:
|
|
env[key] = val
|
|
proc = await asyncio.create_subprocess_exec(
|
|
ironclaw_binary, "--no-onboard",
|
|
stdin=asyncio.subprocess.DEVNULL,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
env=env,
|
|
)
|
|
base_url = f"http://127.0.0.1:{gateway_port}"
|
|
try:
|
|
await wait_for_ready(f"{base_url}/api/health", timeout=60)
|
|
yield base_url
|
|
except TimeoutError:
|
|
# Dump stderr so CI logs show why the server failed to start
|
|
returncode = proc.returncode
|
|
stderr_bytes = b""
|
|
if proc.stderr:
|
|
try:
|
|
stderr_bytes = await asyncio.wait_for(proc.stderr.read(8192), timeout=2)
|
|
except (asyncio.TimeoutError, Exception):
|
|
pass
|
|
stderr_text = stderr_bytes.decode("utf-8", errors="replace")
|
|
proc.kill()
|
|
pytest.fail(
|
|
f"ironclaw server failed to start on port {gateway_port} "
|
|
f"(returncode={returncode}).\nstderr:\n{stderr_text}"
|
|
)
|
|
finally:
|
|
if proc.returncode is None:
|
|
proc.send_signal(signal.SIGTERM)
|
|
try:
|
|
await asyncio.wait_for(proc.wait(), timeout=5)
|
|
except asyncio.TimeoutError:
|
|
proc.kill()
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
async def browser(ironclaw_server):
|
|
"""Session-scoped Playwright browser instance.
|
|
|
|
Reuses a single browser process across all tests. Individual tests
|
|
get isolated contexts via the ``page`` fixture.
|
|
"""
|
|
from playwright.async_api import async_playwright
|
|
|
|
headless = os.environ.get("HEADED", "").strip() not in ("1", "true")
|
|
async with async_playwright() as p:
|
|
b = await p.chromium.launch(headless=headless)
|
|
yield b
|
|
await b.close()
|
|
|
|
|
|
@pytest.fixture
|
|
async def page(ironclaw_server, browser):
|
|
"""Fresh Playwright browser context + page, navigated to the gateway with auth."""
|
|
context = await browser.new_context(viewport={"width": 1280, "height": 720})
|
|
pg = await context.new_page()
|
|
await pg.goto(f"{ironclaw_server}/?token={AUTH_TOKEN}")
|
|
# Wait for the app to initialize (auth screen hidden, SSE connected)
|
|
await pg.wait_for_selector("#auth-screen", state="hidden", timeout=15000)
|
|
yield pg
|
|
await context.close()
|