From 26b1c5b493269d7e2b452d3ab56ba6c8e323c182 Mon Sep 17 00:00:00 2001 From: dingo Date: Sun, 29 Mar 2026 13:16:20 +0700 Subject: [PATCH] docs: mesh cluster documentation, SVG diagrams, and ironclaw->optimclaw rename - MESH_CLUSTER.md: full documentation for autonomous AI mesh network - LAZY_TOOLS.md: lazy tool loading for smaller LLMs - mesh-architecture.svg: colorful network topology diagram - task-routing.svg: scoring algorithm visualization - pq-handshake.svg: ML-KEM-768 key exchange sequence diagram - HTML docs with dark theme styling - Renamed ironclaw references to optimclaw in .env.example and README Co-Authored-By: Claude Opus 4.6 (1M context) --- .env.example | 30 +- README.md | 46 +++ docs/LAZY_TOOLS.md | 126 ++++++ docs/MESH_CLUSTER.md | 447 +++++++++++++++++++++ docs/mesh-architecture.svg | 112 ++++++ docs/pdf/lazy-tools.html | 316 +++++++++++++++ docs/pdf/mesh-cluster.html | 786 +++++++++++++++++++++++++++++++++++++ docs/pq-handshake.svg | 74 ++++ docs/task-routing.svg | 56 +++ 9 files changed, 1978 insertions(+), 15 deletions(-) create mode 100644 docs/LAZY_TOOLS.md create mode 100644 docs/MESH_CLUSTER.md create mode 100644 docs/mesh-architecture.svg create mode 100644 docs/pdf/lazy-tools.html create mode 100644 docs/pdf/mesh-cluster.html create mode 100644 docs/pq-handshake.svg create mode 100644 docs/task-routing.svg diff --git a/.env.example b/.env.example index ce3e3124..a2574196 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,5 @@ # Database Configuration -DATABASE_URL=postgres://localhost/ironclaw +DATABASE_URL=postgres://localhost/optimclaw DATABASE_POOL_SIZE=10 # LLM Provider @@ -26,19 +26,19 @@ DATABASE_POOL_SIZE=10 # === GitHub Copilot === # Uses the OAuth token from your Copilot IDE sign-in (for example -# ~/.config/github-copilot/apps.json on Linux/macOS), or run `ironclaw onboard` +# ~/.config/github-copilot/apps.json on Linux/macOS), or run `optimclaw onboard` # and choose the GitHub device login flow. # LLM_BACKEND=github_copilot # GITHUB_COPILOT_TOKEN=gho_... # GITHUB_COPILOT_MODEL=gpt-4o -# IronClaw injects standard VS Code Copilot headers automatically. +# OptimClaw injects standard VS Code Copilot headers automatically. # Optional advanced headers for custom overrides: # GITHUB_COPILOT_EXTRA_HEADERS=Copilot-Integration-Id:vscode-chat # === NEAR AI (Chat Completions API) === # Two auth modes: # 1. Session token (default): Uses browser OAuth (GitHub/Google) on first run. -# Session token stored in ~/.ironclaw/session.json automatically. +# Session token stored in ~/.optimclaw/session.json automatically. # Base URL defaults to https://private.near.ai # 2. API key: Set NEARAI_API_KEY to use API key auth from cloud.near.ai. # Base URL defaults to https://cloud-api.near.ai @@ -46,7 +46,7 @@ NEARAI_MODEL=Qwen/Qwen3.5-122B-A10B NEARAI_BASE_URL=https://private.near.ai NEARAI_AUTH_URL=https://private.near.ai # NEARAI_SESSION_TOKEN=sess_... # hosting providers: set this -# NEARAI_SESSION_PATH=~/.ironclaw/session.json # optional, default shown +# NEARAI_SESSION_PATH=~/.optimclaw/session.json # optional, default shown # NEARAI_API_KEY=... # API key from cloud.near.ai # Local LLM Providers (Ollama, LM Studio, vLLM, LiteLLM) @@ -63,7 +63,7 @@ NEARAI_AUTH_URL=https://private.near.ai # LLM_API_KEY=sk-... # optional for local servers # Custom HTTP headers for OpenAI-compatible providers # Format: comma-separated key:value pairs -# LLM_EXTRA_HEADERS=HTTP-Referer:https://github.com/nearai/ironclaw,X-Title:ironclaw +# LLM_EXTRA_HEADERS=HTTP-Referer:https://github.com/nearai/optimclaw,X-Title:optimclaw # === OpenRouter (300+ models via OpenAI-compatible) === # LLM_MODEL=anthropic/claude-sonnet-4 # see openrouter.ai/models for IDs @@ -145,7 +145,7 @@ HTTP_HOST=0.0.0.0 HTTP_PORT=8080 HTTP_WEBHOOK_SECRET=your-webhook-secret # Webhook authentication uses HMAC-SHA256 signature verification. -# Callers must send an X-IronClaw-Signature header with format: sha256= +# Callers must send an X-OptimClaw-Signature header with format: sha256= # where the digest is HMAC-SHA256(HTTP_WEBHOOK_SECRET, raw_request_body) in lowercase hex. # # Example (bash): @@ -153,7 +153,7 @@ HTTP_WEBHOOK_SECRET=your-webhook-secret # SIG=$(echo -n "$BODY" | openssl dgst -sha256 -hmac "$HTTP_WEBHOOK_SECRET" | cut -d' ' -f2) # curl -X POST http://localhost:8080/webhook \ # -H "Content-Type: application/json" \ -# -H "X-IronClaw-Signature: sha256=$SIG" \ +# -H "X-OptimClaw-Signature: sha256=$SIG" \ # -d "$BODY" # # DEPRECATED: Passing "secret" in the JSON body still works but will be removed in a future release. @@ -170,7 +170,7 @@ HTTP_WEBHOOK_SECRET=your-webhook-secret # SIGNAL_IGNORE_STORIES=true # Agent Settings -AGENT_NAME=ironclaw +AGENT_NAME=optimclaw AGENT_MAX_PARALLEL_JOBS=5 AGENT_JOB_TIMEOUT_SECS=3600 AGENT_STUCK_THRESHOLD_SECS=300 @@ -205,7 +205,7 @@ HEARTBEAT_NOTIFY_USER=default # # commands directly on the host. Without this # # set to "true", full_access is downgraded to # # workspace_write. -# SANDBOX_IMAGE=ironclaw-worker:latest +# SANDBOX_IMAGE=optimclaw-worker:latest # SANDBOX_TIMEOUT_SECS=120 # SANDBOX_MEMORY_LIMIT_MB=2048 @@ -214,11 +214,11 @@ SAFETY_MAX_OUTPUT_LENGTH=100000 SAFETY_INJECTION_CHECK_ENABLED=true # Restart Feature (Docker containers only) -# Set IRONCLAW_IN_DOCKER=true in the container entrypoint to enable the restart feature. +# Set OPTIMCLAW_IN_DOCKER=true in the container entrypoint to enable the restart feature. # Without this, the restart tool and /restart command will be disabled. -# IRONCLAW_IN_DOCKER=false -# IRONCLAW_RESTART_DELAY=5 # default wait before exit (seconds, range: 1-30) -# IRONCLAW_MAX_FAILURES=10 # max consecutive failures before container exits +# OPTIMCLAW_IN_DOCKER=false +# OPTIMCLAW_RESTART_DELAY=5 # default wait before exit (seconds, range: 1-30) +# OPTIMCLAW_MAX_FAILURES=10 # max consecutive failures before container exits # Logging -RUST_LOG=ironclaw=debug,tower_http=debug +RUST_LOG=optimclaw=debug,tower_http=debug diff --git a/README.md b/README.md index a4882c49..fa021253 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@

Philosophy • Features • + Mesh Cluster • + Lazy Tools • Installation • Configuration • Security • @@ -79,6 +81,50 @@ OptimClaw is the AI assistant you can actually trust with your personal and prof - **Workspace Filesystem** - Flexible path-based storage for notes, logs, and context - **Identity Files** - Maintain consistent personality and preferences across sessions +## Mesh Cluster + +OptimClaw instances can form an **autonomous AI mesh network** where nodes discover each other automatically, coordinate via a gossip protocol, and route tasks intelligently across the cluster. + +Key highlights: + +- **Zero-config discovery** -- UDP beacon broadcast finds peers on the local network automatically +- **Post-quantum encryption** -- ML-KEM-768 key exchange with AES-256-GCM authenticated encryption protects all inter-node traffic against both classical and quantum adversaries +- **SWIM gossip membership** -- Reliable failure detection and cluster state convergence in O(log N) rounds +- **Intelligent task routing** -- A scoring algorithm balances load, latency, capability match, session affinity, and region locality to pick the best node for each task +- **Graceful degradation** -- Nodes operate independently if connectivity is lost; no split-brain data corruption + +Quick start (two nodes on one machine): + +```bash +# Terminal 1 +export CLUSTER_ENABLED=true CLUSTER_SECRET="your-32-char-secret-here-change-me" CLUSTER_NODE_ID=node-a +cargo run + +# Terminal 2 +export CLUSTER_ENABLED=true CLUSTER_SECRET="your-32-char-secret-here-change-me" CLUSTER_NODE_ID=node-b CLUSTER_BIND_PORT=9410 +cargo run +``` + +Monitor via `GET /api/mesh/status` and `GET /api/mesh/nodes`. + +See [docs/MESH_CLUSTER.md](docs/MESH_CLUSTER.md) for the full guide covering architecture, configuration reference, security model, and troubleshooting. + +## Lazy Tools + +Lazy tool loading reduces the system prompt from approximately 13,000 tokens to approximately 4,000 tokens by deferring tool schemas that are not immediately needed. + +Enable it with: + +```bash +export OPTIMCLAW_LAZY_TOOLS=1 +``` + +When enabled, 12 core tools (echo, time, json, http, web_fetch, file_read, file_write, shell, memory_search, memory_write, message, tool_info) are loaded eagerly. All other tools -- including MCP, WASM, and skill tools -- are listed by name only. The LLM calls `tool_info` to load the full schema for any additional tool on demand. + +This is recommended for production deployments and cost-sensitive usage with expensive models. + +See [docs/LAZY_TOOLS.md](docs/LAZY_TOOLS.md) for the full guide. + ## Installation ### Prerequisites diff --git a/docs/LAZY_TOOLS.md b/docs/LAZY_TOOLS.md new file mode 100644 index 00000000..b079a366 --- /dev/null +++ b/docs/LAZY_TOOLS.md @@ -0,0 +1,126 @@ +# OptimClaw Lazy Tool Loading + +## Overview + +Lazy tool loading is an optimization that dramatically reduces the initial system prompt size sent to the LLM. Instead of injecting full JSON schemas for every available tool into each request, OptimClaw loads only a small core set of tools eagerly and defers the rest until the LLM requests them. + +**Impact:** System prompt size drops from approximately 13,000 tokens to approximately 4,000 tokens -- a 70% reduction. This saves cost on every LLM call and leaves more of the context window available for conversation history and tool outputs. + +## How to Enable + +Set the environment variable: + +```bash +export OPTIMCLAW_LAZY_TOOLS=1 +``` + +Or add it to `~/.optimclaw/.env`: + +```env +OPTIMCLAW_LAZY_TOOLS=1 +``` + +To disable (default behavior -- all tools loaded eagerly): + +```bash +export OPTIMCLAW_LAZY_TOOLS=0 +# or simply unset it +unset OPTIMCLAW_LAZY_TOOLS +``` + +## Core Tools (Always Loaded) + +When lazy loading is enabled, the following 12 core tools are always included in the system prompt. These are the tools the LLM needs most frequently and cover the essential interaction patterns: + +| # | Tool | Purpose | +|---|------|---------| +| 1 | `echo` | Return text to the user | +| 2 | `time` | Get current date and time | +| 3 | `json` | Parse and query JSON data | +| 4 | `http` | Make HTTP requests to allowed endpoints | +| 5 | `web_fetch` | Fetch and extract content from web pages | +| 6 | `file_read` | Read files from the workspace | +| 7 | `file_write` | Write files to the workspace | +| 8 | `shell` | Execute shell commands in the sandbox | +| 9 | `memory_search` | Search persistent memory (hybrid FTS + vector) | +| 10 | `memory_write` | Write to persistent memory | +| 11 | `message` | Send messages to channels | +| 12 | `tool_info` | Discover and load additional tools on demand | + +## Tool Discovery with tool_info + +The `tool_info` tool is the mechanism by which the LLM discovers and loads deferred tools. When the LLM determines it needs a tool that is not in its current context, it calls `tool_info` to retrieve the full schema. + +### How It Works + +1. The system prompt includes a brief note listing the names of all available (but not yet loaded) tools. +2. When the LLM needs one of these tools, it calls `tool_info` with the tool name or a search query. +3. `tool_info` returns the full JSON schema (parameters, description, examples) for the matched tools. +4. The LLM can then call the newly loaded tool in subsequent turns. + +### tool_info Parameters + +```json +{ + "name": "tool_info", + "parameters": { + "query": { + "type": "string", + "description": "Exact tool name or keyword search query" + }, + "max_results": { + "type": "number", + "description": "Maximum tools to return (default: 5)" + } + } +} +``` + +### Example Flow + +**System prompt includes:** +> Additional tools available (use `tool_info` to load): `job_create`, `job_status`, `job_cancel`, `routine_create`, `routine_list`, `skill_search`, `skill_install`, `extension_install`, `secrets_set`, `secrets_get`, ... + +**LLM decides it needs to create a background job:** + +``` +LLM -> tool_info(query="job_create") + +tool_info returns: +{ + "tools": [{ + "name": "job_create", + "description": "Create a new background job with the given prompt and priority", + "parameters": { + "prompt": { "type": "string", "required": true }, + "priority": { "type": "number", "default": 5 }, + "timeout_secs": { "type": "number", "default": 300 } + } + }] +} + +LLM -> job_create(prompt="Summarize today's news", priority=3) +``` + +## When to Use Lazy Loading + +| Scenario | Recommendation | +|----------|----------------| +| Production deployment with many tools/MCP servers | Enable -- significant token savings | +| Development and debugging | Disable -- easier to see all available tools | +| Cost-sensitive usage with expensive models | Enable -- reduces per-request cost | +| Clusters with heterogeneous tool sets | Enable -- each node may have different tools | +| Simple setups with few tools (<15 total) | Either -- minimal difference | + +## Performance Characteristics + +- **First request:** Faster, because the system prompt is smaller and the LLM processes fewer tokens. +- **Tool discovery round-trip:** When the LLM calls `tool_info`, it adds one extra turn before the actual tool call. In practice this is rare because the 12 core tools handle the majority of interactions. +- **Subsequent requests in the same session:** Tool schemas loaded via `tool_info` remain in the conversation context for the duration of the session, so discovery cost is paid at most once per tool per session. + +## Interaction with Other Features + +- **MCP tools:** MCP-connected tool schemas are also deferred when lazy loading is enabled. They appear in the "additional tools available" list and are loaded via `tool_info`. +- **WASM tools:** Same behavior as MCP tools -- deferred and discoverable. +- **Skills:** Skill tools (`skill_list`, `skill_search`, `skill_install`, `skill_remove`) are deferred. The skill system itself is unaffected. +- **Mesh cluster:** Lazy loading is a per-node setting. Different nodes in a cluster can have different settings. diff --git a/docs/MESH_CLUSTER.md b/docs/MESH_CLUSTER.md new file mode 100644 index 00000000..905962c6 --- /dev/null +++ b/docs/MESH_CLUSTER.md @@ -0,0 +1,447 @@ +# OptimClaw Mesh Cluster + +## Overview + +The OptimClaw Mesh Cluster enables multiple OptimClaw instances to form an autonomous AI mesh network. Each node in the cluster operates independently while collaborating on tasks, sharing workload, and providing fault tolerance. Nodes discover each other automatically via UDP beacons, authenticate using post-quantum cryptography, and coordinate through a gossip-based membership protocol. + +Key capabilities: + +- **Automatic discovery** -- zero-configuration node joining via UDP broadcast beacons +- **Post-quantum security** -- ML-KEM-768 key encapsulation with AES-256-GCM authenticated encryption +- **Gossip membership** -- SWIM protocol for reliable failure detection and cluster state convergence +- **Intelligent task routing** -- scoring algorithm that considers load, latency, capability, and affinity +- **Graceful degradation** -- nodes operate independently if connectivity is lost + +## Architecture + +``` + ┌─────────────────────────────────────────────┐ + │ Mesh Cluster │ + │ │ + ┌──────────────┐ │ ┌──────────┐ Gossip ┌──────────┐ │ + │ Client │──────►│ │ Node A │◄────────────►│ Node B │ │ + │ (any chan.) │ │ │ │ (SWIM) │ │ │ + └──────────────┘ │ │ ┌──────┐ │ │ ┌──────┐ │ │ + │ │ │Agent │ │ │ │Agent │ │ │ + │ │ │ Loop │ │ │ │ Loop │ │ │ + │ │ └──────┘ │ │ └──────┘ │ │ + │ │ ┌──────┐ │ │ ┌──────┐ │ │ + │ │ │Tools │ │ │ │Tools │ │ │ + │ │ └──────┘ │ │ └──────┘ │ │ + │ └─────┬────┘ └────┬─────┘ │ + │ │ │ │ + │ │ UDP Beacons │ │ + │ │◄───────────────────────►│ │ + │ │ │ │ + │ │ Task Routing │ │ + │ │◄───────────────────────►│ │ + │ │ (ML-KEM-768 + │ │ + │ │ AES-256-GCM) │ │ + │ ┌─────┴────┐ ┌────┴─────┐ │ + │ │ Node C │◄────────────►│ Node D │ │ + │ └──────────┘ Gossip └──────────┘ │ + │ │ + └─────────────────────────────────────────────┘ + +Data flow: + 1. UDP beacon broadcast → node discovery + 2. ML-KEM-768 handshake → shared secret + 3. AES-256-GCM encrypted channel established + 4. SWIM gossip protocol → membership state + 5. Task routing → best node selected via scoring + 6. Encrypted task dispatch + result collection +``` + +## Configuration + +All cluster settings are controlled via environment variables prefixed with `CLUSTER_`. They can be set in `~/.optimclaw/.env` or passed directly. + +| Variable | Type | Default | Description | +|----------|------|---------|-------------| +| `CLUSTER_ENABLED` | bool | `false` | Enable mesh cluster mode | +| `CLUSTER_NODE_ID` | string | auto (hostname) | Unique identifier for this node | +| `CLUSTER_BIND_ADDR` | string | `0.0.0.0` | Address to bind the cluster transport | +| `CLUSTER_BIND_PORT` | u16 | `9400` | Port for the encrypted cluster transport | +| `CLUSTER_BEACON_PORT` | u16 | `9401` | UDP port for discovery beacons | +| `CLUSTER_BEACON_INTERVAL_MS` | u64 | `5000` | Milliseconds between beacon broadcasts | +| `CLUSTER_BEACON_SUBNET` | string | `255.255.255.255` | Broadcast address for beacons | +| `CLUSTER_SECRET` | string | *required* | Pre-shared key for cluster authentication (min 32 chars) | +| `CLUSTER_SEEDS` | string | `""` | Comma-separated seed node addresses (`host:port`) for non-broadcast environments | +| `CLUSTER_GOSSIP_INTERVAL_MS` | u64 | `1000` | Milliseconds between gossip protocol rounds | +| `CLUSTER_GOSSIP_FANOUT` | u8 | `3` | Number of peers to gossip with per round | +| `CLUSTER_SUSPICION_MULT` | u8 | `4` | Multiplier for suspicion timeout (suspicion_mult * gossip_interval) | +| `CLUSTER_PROBE_INTERVAL_MS` | u64 | `2000` | Milliseconds between SWIM probe pings | +| `CLUSTER_PROBE_TIMEOUT_MS` | u64 | `500` | Timeout for a direct probe response | +| `CLUSTER_INDIRECT_PROBES` | u8 | `3` | Number of indirect probes before suspicion | +| `CLUSTER_TASK_TIMEOUT_SECS` | u64 | `300` | Timeout for a routed task to complete | +| `CLUSTER_MAX_NODES` | u16 | `64` | Maximum cluster size | +| `CLUSTER_TLS_CERT` | path | `""` | Optional TLS certificate for cross-datacenter transport | +| `CLUSTER_TLS_KEY` | path | `""` | Optional TLS private key | +| `CLUSTER_ADVERTISE_ADDR` | string | auto | Address advertised to other nodes (for NAT traversal) | +| `CLUSTER_ADVERTISE_PORT` | u16 | same as bind | Port advertised to other nodes | +| `CLUSTER_REGION` | string | `""` | Logical region tag for locality-aware routing | +| `CLUSTER_CAPABILITIES` | string | `""` | Comma-separated capability tags (e.g., `gpu,high-memory,docker`) | + +### Minimal Configuration + +```env +CLUSTER_ENABLED=true +CLUSTER_SECRET=my-very-long-pre-shared-key-at-least-32-chars +``` + +### Cross-Datacenter Configuration + +```env +CLUSTER_ENABLED=true +CLUSTER_SECRET=my-very-long-pre-shared-key-at-least-32-chars +CLUSTER_SEEDS=dc1-node1.example.com:9400,dc2-node1.example.com:9400 +CLUSTER_ADVERTISE_ADDR=203.0.113.10 +CLUSTER_REGION=us-east-1 +CLUSTER_TLS_CERT=/etc/optimclaw/cluster.crt +CLUSTER_TLS_KEY=/etc/optimclaw/cluster.key +``` + +## Discovery Protocol + +Nodes discover each other using a UDP beacon protocol. When a node starts with `CLUSTER_ENABLED=true`, it begins broadcasting beacon packets on the configured broadcast address and port. + +### Beacon Packet Format + +``` +Offset Size Field +0 4 Magic bytes: 0x4F 0x43 0x4D 0x53 ("OCMS") +4 1 Protocol version (currently 0x01) +5 2 Beacon port (big-endian u16) +7 2 Transport port (big-endian u16) +9 32 Node ID (UTF-8, zero-padded) +41 32 HMAC-SHA256 of bytes 0..41 using CLUSTER_SECRET +``` + +Total beacon size: 73 bytes. + +### Discovery Sequence + +1. On startup, the node broadcasts a beacon every `CLUSTER_BEACON_INTERVAL_MS` milliseconds to `CLUSTER_BEACON_SUBNET:CLUSTER_BEACON_PORT`. +2. All listening nodes receive the beacon, verify the HMAC against their own `CLUSTER_SECRET`, and extract the sender's transport address. +3. If the beacon is from an unknown node, the receiving node initiates a post-quantum key exchange (see below) over TCP to the sender's transport address. +4. Once the encrypted channel is established, the new node is added to the membership list and the gossip protocol takes over. +5. In non-broadcast environments (cloud, cross-datacenter), set `CLUSTER_SEEDS` to bootstrap. The node will contact seed addresses directly instead of relying on broadcast. + +Beacons continue to be sent after joining to help new nodes discover the cluster. + +## Post-Quantum Cryptography + +All inter-node communication is encrypted using a hybrid post-quantum scheme to protect against both classical and quantum adversaries. + +### Key Exchange: ML-KEM-768 + +ML-KEM-768 (formerly CRYSTALS-Kyber) is a lattice-based key encapsulation mechanism standardized in FIPS 203. It provides IND-CCA2 security at NIST security level 3 (roughly equivalent to AES-192). + +The handshake proceeds as follows: + +1. **Initiator** generates an ML-KEM-768 keypair (ephemeral) and sends the public key (1184 bytes) along with its node ID and a challenge derived from `CLUSTER_SECRET`. +2. **Responder** verifies the challenge, encapsulates a shared secret using the received public key, and sends back the ciphertext (1088 bytes) along with its own challenge response. +3. Both sides derive the same 256-bit shared secret from the ML-KEM decapsulation. +4. The shared secret is combined with `CLUSTER_SECRET` via HKDF-SHA256 to produce the final session key, binding the session to the cluster identity. + +### Authenticated Encryption: AES-256-GCM + +All messages after the handshake are encrypted with AES-256-GCM using the derived session key: + +- **Nonce**: 96-bit, incremented per message (with sender-direction bit to avoid reuse) +- **AAD (Additional Authenticated Data)**: message type + sequence number + sender node ID +- **Tag**: 128-bit authentication tag appended to ciphertext + +### Key Rotation + +Session keys are rotated every 1 hour or after 2^32 messages, whichever comes first. Rotation uses a new ML-KEM-768 encapsulation within the existing encrypted channel. + +### Why Post-Quantum? + +Mesh clusters may carry sensitive task data (credentials, personal information, tool outputs). Harvest-now-decrypt-later attacks make it prudent to deploy post-quantum cryptography today, even before large-scale quantum computers exist. + +## Gossip Protocol (SWIM Membership) + +The cluster uses the SWIM (Scalable Weakly-consistent Infection-style process group Membership) protocol for membership management and failure detection. + +### Membership States + +Each node maintains a membership list where every entry is in one of three states: + +| State | Meaning | +|-------|---------| +| **Alive** | Node is healthy and responsive | +| **Suspect** | Node failed to respond to probes; may be down | +| **Dead** | Node confirmed unreachable; removed from routing | + +### Protocol Rounds + +Every `CLUSTER_GOSSIP_INTERVAL_MS`, each node performs: + +1. **Probe** -- Select a random alive member and send a direct ping. If no ack within `CLUSTER_PROBE_TIMEOUT_MS`, send indirect pings through `CLUSTER_INDIRECT_PROBES` random members. If still no ack, mark the target as Suspect. +2. **Gossip** -- Piggyback membership updates (state changes, join/leave events) on probe messages. Each update includes a Lamport timestamp for crdt-style conflict resolution. +3. **Suspicion** -- Suspect nodes have `CLUSTER_SUSPICION_MULT * CLUSTER_GOSSIP_INTERVAL_MS` to refute by sending an Alive message with a higher incarnation number. If not refuted, the node transitions to Dead. + +### Consistency + +SWIM provides eventual consistency. After a state change, all nodes converge within O(log N) gossip rounds, where N is the cluster size. With default settings (1s gossip interval, fanout 3), a 64-node cluster converges in under 7 seconds. + +### Join and Leave + +- **Join**: Triggered by beacon discovery or seed contact. The joining node sends a Join message; existing members propagate the new membership via gossip. +- **Graceful leave**: A node sends a Leave message before shutting down. Other nodes immediately mark it Dead without suspicion. +- **Crash**: Detected by the probe/suspicion mechanism described above. + +## Task Routing Algorithm + +When a task arrives at any node, the router decides whether to execute it locally or forward it to a better-suited node. The decision is based on a scoring formula applied to each alive node. + +### Scoring Formula + +``` +score(node) = w_load * (1 - load_ratio) + + w_latency * (1 - latency_ratio) + + w_capability * capability_match + + w_affinity * affinity_bonus + + w_locality * locality_bonus +``` + +Where: + +| Factor | Weight (default) | Description | +|--------|-------------------|-------------| +| `load_ratio` | `w_load = 0.35` | Current jobs / max parallel jobs (lower is better) | +| `latency_ratio` | `w_latency = 0.25` | P95 RTT to this node / max observed RTT (lower is better) | +| `capability_match` | `w_capability = 0.25` | 1.0 if node has all required capabilities, 0.0 otherwise | +| `affinity_bonus` | `w_affinity = 0.10` | 1.0 if the task has session affinity to this node, 0.0 otherwise | +| `locality_bonus` | `w_locality = 0.05` | 1.0 if same `CLUSTER_REGION`, 0.5 if no region set, 0.0 otherwise | + +### Routing Decision + +1. Compute `score(node)` for all alive nodes including self. +2. If the local node's score is within 10% of the best score, execute locally (avoids unnecessary forwarding overhead). +3. Otherwise, forward the task to the highest-scoring node over the encrypted channel. +4. If the target node fails to accept within 5 seconds, fall back to local execution. +5. Results are returned to the originating node and delivered to the original client. + +### Session Affinity + +Tasks that reference an ongoing conversation or job context are preferentially routed to the node that holds that context. This avoids expensive context transfer between nodes. + +## API Endpoints + +The mesh cluster exposes monitoring endpoints on the standard web gateway. + +### GET /api/mesh/status + +Returns the cluster status for the local node. + +**Response:** + +```json +{ + "cluster_enabled": true, + "node_id": "node-alpha", + "state": "alive", + "region": "us-east-1", + "capabilities": ["gpu", "docker"], + "uptime_secs": 86423, + "transport": { + "bind_addr": "0.0.0.0:9400", + "advertise_addr": "203.0.113.10:9400", + "encryption": "ML-KEM-768 + AES-256-GCM", + "protocol_version": 1 + }, + "membership": { + "alive": 4, + "suspect": 0, + "dead": 1, + "total_seen": 5 + }, + "routing": { + "local_load": 0.35, + "tasks_routed_out": 142, + "tasks_routed_in": 87, + "tasks_failed_over": 3 + } +} +``` + +### GET /api/mesh/nodes + +Returns the membership list with per-node details. + +**Response:** + +```json +{ + "nodes": [ + { + "node_id": "node-alpha", + "state": "alive", + "addr": "203.0.113.10:9400", + "region": "us-east-1", + "capabilities": ["gpu", "docker"], + "load_ratio": 0.35, + "latency_ms": 0, + "last_seen": "2026-03-29T12:34:56Z", + "incarnation": 7, + "is_self": true + }, + { + "node_id": "node-beta", + "state": "alive", + "addr": "203.0.113.11:9400", + "region": "us-east-1", + "capabilities": ["high-memory"], + "load_ratio": 0.12, + "latency_ms": 2, + "last_seen": "2026-03-29T12:34:55Z", + "incarnation": 3, + "is_self": false + } + ] +} +``` + +### POST /api/mesh/nodes/{node_id}/drain + +Puts a node into drain mode (stops accepting new routed tasks, finishes existing ones). Useful before maintenance. + +**Response:** + +```json +{ + "node_id": "node-beta", + "drained": true, + "remaining_tasks": 2 +} +``` + +## Quick Start + +### Running Two Nodes on the Same Machine + +**Terminal 1 (Node A):** + +```bash +export CLUSTER_ENABLED=true +export CLUSTER_SECRET="change-me-to-a-strong-shared-secret-at-least-32-characters" +export CLUSTER_NODE_ID=node-a +export CLUSTER_BIND_PORT=9400 +export CLUSTER_BEACON_PORT=9401 +export DATABASE_URL=postgres://localhost/optimclaw_a + +optimclaw onboard # if not already configured +cargo run +``` + +**Terminal 2 (Node B):** + +```bash +export CLUSTER_ENABLED=true +export CLUSTER_SECRET="change-me-to-a-strong-shared-secret-at-least-32-characters" +export CLUSTER_NODE_ID=node-b +export CLUSTER_BIND_PORT=9410 +export CLUSTER_BEACON_PORT=9401 # same beacon port so they discover each other +export DATABASE_URL=postgres://localhost/optimclaw_b + +optimclaw onboard +cargo run +``` + +Within 5 seconds, both nodes should discover each other via UDP beacons. Verify by hitting the status endpoint: + +```bash +curl http://localhost:3000/api/mesh/status | jq .membership +# {"alive": 2, "suspect": 0, "dead": 0, "total_seen": 2} +``` + +### Running Across Machines + +On each machine, set the same `CLUSTER_SECRET` and either: + +- Ensure UDP broadcast works on the local network (same subnet), or +- Set `CLUSTER_SEEDS` to the address of at least one other node: + +```bash +export CLUSTER_SEEDS=192.168.1.100:9400 +``` + +## Security Model + +### Threat Model + +The mesh cluster is designed to be secure against: + +1. **Passive eavesdropping** -- All traffic is encrypted with AES-256-GCM. +2. **Active MITM** -- The ML-KEM-768 handshake is bound to `CLUSTER_SECRET`, preventing interception by parties without the pre-shared key. +3. **Quantum adversaries** -- ML-KEM-768 provides post-quantum security for key exchange. +4. **Rogue node injection** -- Beacons are authenticated with HMAC-SHA256; the handshake requires `CLUSTER_SECRET`. +5. **Replay attacks** -- Nonces are strictly monotonic; replayed messages are rejected. +6. **Partition exploitation** -- Nodes degrade to independent operation; no split-brain data corruption. + +### Trust Boundaries + +- All nodes sharing the same `CLUSTER_SECRET` are in the same trust domain. +- A compromised `CLUSTER_SECRET` means any attacker can join the cluster. Rotate the secret and restart all nodes if a compromise is suspected. +- Task data (including tool outputs) is encrypted in transit but available in plaintext to any node in the cluster. Do not add untrusted machines to a cluster that handles sensitive data. + +### Network Recommendations + +| Deployment | Recommendation | +|------------|----------------| +| Same LAN | UDP beacons work out of the box. Use a firewall to restrict beacon and transport ports to trusted hosts. | +| Cross-datacenter | Use `CLUSTER_SEEDS`, disable beacons by setting `CLUSTER_BEACON_INTERVAL_MS=0`, enable TLS (`CLUSTER_TLS_CERT` / `CLUSTER_TLS_KEY`), and restrict access via network ACLs. | +| Cloud (AWS/GCP/Azure) | Use private VPC networking. Set `CLUSTER_ADVERTISE_ADDR` to the private IP. Use security groups to restrict ports 9400-9401. | + +## Troubleshooting + +### Nodes not discovering each other + +1. **Check `CLUSTER_SECRET`** -- Must be identical on all nodes. Even trailing whitespace matters. +2. **Check beacon port** -- All nodes must use the same `CLUSTER_BEACON_PORT`. +3. **Check UDP broadcast** -- Some cloud providers and corporate networks block UDP broadcast. Use `CLUSTER_SEEDS` instead. +4. **Check firewall** -- Ports `CLUSTER_BEACON_PORT` (UDP) and `CLUSTER_BIND_PORT` (TCP) must be open. +5. **Check logs** -- Run with `RUST_LOG=optimclaw::cluster=debug` to see beacon send/receive events. + +### Node stuck in Suspect state + +- This typically means the node is slow to respond to probes. +- Increase `CLUSTER_PROBE_TIMEOUT_MS` on busy nodes. +- Increase `CLUSTER_SUSPICION_MULT` to give more time before declaring a node dead. +- Check if the node is CPU-starved or under heavy I/O load. + +### High task routing latency + +- Check `curl localhost:3000/api/mesh/nodes | jq '.nodes[].latency_ms'` to identify slow links. +- Tasks are only routed away from the local node if a remote node scores >10% better. If most tasks should stay local, this is expected behavior. +- For cross-datacenter deployments, set `CLUSTER_REGION` on each node so the locality bonus keeps tasks close. + +### Session key negotiation failures + +- Both nodes must support the same protocol version. Ensure all nodes are running the same OptimClaw release. +- If using TLS (`CLUSTER_TLS_CERT`), verify the certificate is valid and trusted by the other node. +- Check for clock skew greater than 5 minutes between nodes. + +### Node rejoining after network partition + +- After a partition heals, the previously-dead node sends beacons again and is rediscovered. +- The rejoining node increments its incarnation number to override the Dead state in other nodes' membership lists. +- Any tasks that were in-flight to the partitioned node will have timed out and been retried locally. + +### Diagnostic Commands + +```bash +# Check cluster status +curl -s http://localhost:3000/api/mesh/status | jq . + +# List all known nodes +curl -s http://localhost:3000/api/mesh/nodes | jq . + +# Drain a node before maintenance +curl -s -X POST http://localhost:3000/api/mesh/nodes/node-beta/drain | jq . + +# Watch cluster events in real time +RUST_LOG=optimclaw::cluster=debug cargo run 2>&1 | grep cluster +``` diff --git a/docs/mesh-architecture.svg b/docs/mesh-architecture.svg new file mode 100644 index 00000000..47b8ac1c --- /dev/null +++ b/docs/mesh-architecture.svg @@ -0,0 +1,112 @@ + + + + + + + + + + + + + + + + + + + + + + + + OptimClaw Mesh Network + Autonomous Peer-to-Peer AI Agent Cluster with Post-Quantum Encryption + + + + + + + + + + + + + + + ML-KEM-768 + AES-256-GCM + PQ Encrypted + PQ Encrypted + + + + + + + Node A + Laptop | 6GB VRAM + RTX 3060 Mobile + qwen2.5-coder:7b + Load: 35% | 12 tools + + + + + + + Node B + Server | 24GB VRAM + RTX 4090 + qwen3-coder:30b + Load: 10% | 30 tools + + + + + + + Node C + Cloud | 80GB VRAM + A100 + llama3.1:70b + Load: 5% | 30 tools + + + + + + + Node D + Remote | 48GB VRAM + 2x RTX 3090 + deepseek-coder:33b + Load: 72% | 30 tools + + + + + + + Node E + Edge | 8GB VRAM + RTX 4060 + qwen3:8b + Load: 20% | 12 tools + + + + + + Protocol Stack + + UDP Beacon Discovery (port 9900) + ML-KEM-768 Key Exchange + AES-256-GCM Encrypted WebSocket + SWIM Gossip Protocol + + + diff --git a/docs/pdf/lazy-tools.html b/docs/pdf/lazy-tools.html new file mode 100644 index 00000000..3a782144 --- /dev/null +++ b/docs/pdf/lazy-tools.html @@ -0,0 +1,316 @@ + + + + + + + OptimClaw - lazy-tools + + + + +

+

OptimClaw - lazy-tools

+
+

OptimClaw Lazy Tool Loading

+

Overview

+

Lazy tool loading is an optimization that dramatically reduces the +initial system prompt size sent to the LLM. Instead of injecting full +JSON schemas for every available tool into each request, OptimClaw loads +only a small core set of tools eagerly and defers the rest until the LLM +requests them.

+

Impact: System prompt size drops from approximately +13,000 tokens to approximately 4,000 tokens – a 70% reduction. This +saves cost on every LLM call and leaves more of the context window +available for conversation history and tool outputs.

+

How to Enable

+

Set the environment variable:

+
export OPTIMCLAW_LAZY_TOOLS=1
+

Or add it to ~/.optimclaw/.env:

+
OPTIMCLAW_LAZY_TOOLS=1
+

To disable (default behavior – all tools loaded eagerly):

+
export OPTIMCLAW_LAZY_TOOLS=0
+# or simply unset it
+unset OPTIMCLAW_LAZY_TOOLS
+

Core Tools (Always Loaded)

+

When lazy loading is enabled, the following 12 core tools are always +included in the system prompt. These are the tools the LLM needs most +frequently and cover the essential interaction patterns:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
#ToolPurpose
1echoReturn text to the user
2timeGet current date and time
3jsonParse and query JSON data
4httpMake HTTP requests to allowed endpoints
5web_fetchFetch and extract content from web pages
6file_readRead files from the workspace
7file_writeWrite files to the workspace
8shellExecute shell commands in the sandbox
9memory_searchSearch persistent memory (hybrid FTS + vector)
10memory_writeWrite to persistent memory
11messageSend messages to channels
12tool_infoDiscover and load additional tools on demand
+

Tool Discovery with +tool_info

+

The tool_info tool is the mechanism by which the LLM +discovers and loads deferred tools. When the LLM determines it needs a +tool that is not in its current context, it calls tool_info +to retrieve the full schema.

+

How It Works

+
    +
  1. The system prompt includes a brief note listing the names of all +available (but not yet loaded) tools.
    2. +
  2. When the LLM needs one of these tools, it calls +tool_info with the tool name or a search query.
    3. +
  3. tool_info returns the full JSON schema (parameters, +description, examples) for the matched tools.
    4. +
  4. The LLM can then call the newly loaded tool in subsequent +turns.
    5. +
+

tool_info Parameters

+
{
+  "name": "tool_info",
+  "parameters": {
+    "query": {
+      "type": "string",
+      "description": "Exact tool name or keyword search query"
+    },
+    "max_results": {
+      "type": "number",
+      "description": "Maximum tools to return (default: 5)"
+    }
+  }
+}
+

Example Flow

+

System prompt includes: > Additional tools +available (use tool_info to load): job_create, +job_status, job_cancel, +routine_create, routine_list, +skill_search, skill_install, +extension_install, secrets_set, +secrets_get, …

+

LLM decides it needs to create a background job:

+
LLM -> tool_info(query="job_create")
+
+tool_info returns:
+{
+  "tools": [{
+    "name": "job_create",
+    "description": "Create a new background job with the given prompt and priority",
+    "parameters": {
+      "prompt": { "type": "string", "required": true },
+      "priority": { "type": "number", "default": 5 },
+      "timeout_secs": { "type": "number", "default": 300 }
+    }
+  }]
+}
+
+LLM -> job_create(prompt="Summarize today's news", priority=3)
+

When to Use Lazy Loading

+ ++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ScenarioRecommendation
Production deployment with many tools/MCP serversEnable – significant token savings
Development and debuggingDisable – easier to see all available tools
Cost-sensitive usage with expensive modelsEnable – reduces per-request cost
Clusters with heterogeneous tool setsEnable – each node may have different tools
Simple setups with few tools (<15 total)Either – minimal difference
+

Performance Characteristics

+
    +
  • First request: Faster, because the system prompt is +smaller and the LLM processes fewer tokens.
    • +
  • Tool discovery round-trip: When the LLM calls +tool_info, it adds one extra turn before the actual tool +call. In practice this is rare because the 12 core tools handle the +majority of interactions.
    • +
  • Subsequent requests in the same session: Tool +schemas loaded via tool_info remain in the conversation +context for the duration of the session, so discovery cost is paid at +most once per tool per session.
    • +
+

Interaction with Other +Features

+
    +
  • MCP tools: MCP-connected tool schemas are also +deferred when lazy loading is enabled. They appear in the “additional +tools available” list and are loaded via tool_info.
    • +
  • WASM tools: Same behavior as MCP tools – deferred +and discoverable.
    • +
  • Skills: Skill tools (skill_list, +skill_search, skill_install, +skill_remove) are deferred. The skill system itself is +unaffected.
    • +
  • Mesh cluster: Lazy loading is a per-node setting. +Different nodes in a cluster can have different settings.
    • +
+ + diff --git a/docs/pdf/mesh-cluster.html b/docs/pdf/mesh-cluster.html new file mode 100644 index 00000000..65e609f0 --- /dev/null +++ b/docs/pdf/mesh-cluster.html @@ -0,0 +1,786 @@ + + + + + + + OptimClaw - mesh-cluster + + + + +
+

OptimClaw - mesh-cluster

+
+

OptimClaw Mesh Cluster

+

Overview

+

The OptimClaw Mesh Cluster enables multiple OptimClaw instances to +form an autonomous AI mesh network. Each node in the cluster operates +independently while collaborating on tasks, sharing workload, and +providing fault tolerance. Nodes discover each other automatically via +UDP beacons, authenticate using post-quantum cryptography, and +coordinate through a gossip-based membership protocol.

+

Key capabilities:

+
    +
  • Automatic discovery – zero-configuration node +joining via UDP broadcast beacons
    • +
  • Post-quantum security – ML-KEM-768 key +encapsulation with AES-256-GCM authenticated encryption
    • +
  • Gossip membership – SWIM protocol for reliable +failure detection and cluster state convergence
    • +
  • Intelligent task routing – scoring algorithm that +considers load, latency, capability, and affinity
    • +
  • Graceful degradation – nodes operate independently +if connectivity is lost
    • +
+

Architecture

+
                         ┌─────────────────────────────────────────────┐
+                         │              Mesh Cluster                   │
+                         │                                             │
+  ┌──────────────┐       │   ┌──────────┐    Gossip     ┌──────────┐  │
+  │   Client     │──────►│   │  Node A  │◄────────────►│  Node B  │  │
+  │  (any chan.) │       │   │          │   (SWIM)      │          │  │
+  └──────────────┘       │   │ ┌──────┐ │               │ ┌──────┐ │  │
+                         │   │ │Agent │ │               │ │Agent │ │  │
+                         │   │ │ Loop │ │               │ │ Loop │ │  │
+                         │   │ └──────┘ │               │ └──────┘ │  │
+                         │   │ ┌──────┐ │               │ ┌──────┐ │  │
+                         │   │ │Tools │ │               │ │Tools │ │  │
+                         │   │ └──────┘ │               │ └──────┘ │  │
+                         │   └─────┬────┘               └────┬─────┘  │
+                         │         │                         │        │
+                         │         │    UDP Beacons          │        │
+                         │         │◄───────────────────────►│        │
+                         │         │                         │        │
+                         │         │    Task Routing          │        │
+                         │         │◄───────────────────────►│        │
+                         │         │   (ML-KEM-768 +         │        │
+                         │         │    AES-256-GCM)         │        │
+                         │   ┌─────┴────┐               ┌────┴─────┐  │
+                         │   │  Node C  │◄────────────►│  Node D  │  │
+                         │   └──────────┘   Gossip      └──────────┘  │
+                         │                                             │
+                         └─────────────────────────────────────────────┘
+
+Data flow:
+  1. UDP beacon broadcast → node discovery
+  2. ML-KEM-768 handshake → shared secret
+  3. AES-256-GCM encrypted channel established
+  4. SWIM gossip protocol → membership state
+  5. Task routing → best node selected via scoring
+  6. Encrypted task dispatch + result collection
+

Configuration

+

All cluster settings are controlled via environment variables +prefixed with CLUSTER_. They can be set in +~/.optimclaw/.env or passed directly.

+ ++++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
VariableTypeDefaultDescription
CLUSTER_ENABLEDboolfalseEnable mesh cluster mode
CLUSTER_NODE_IDstringauto (hostname)Unique identifier for this node
CLUSTER_BIND_ADDRstring0.0.0.0Address to bind the cluster transport
CLUSTER_BIND_PORTu169400Port for the encrypted cluster transport
CLUSTER_BEACON_PORTu169401UDP port for discovery beacons
CLUSTER_BEACON_INTERVAL_MSu645000Milliseconds between beacon broadcasts
CLUSTER_BEACON_SUBNETstring255.255.255.255Broadcast address for beacons
CLUSTER_SECRETstringrequiredPre-shared key for cluster authentication (min 32 chars)
CLUSTER_SEEDSstring""Comma-separated seed node addresses (host:port) for +non-broadcast environments
CLUSTER_GOSSIP_INTERVAL_MSu641000Milliseconds between gossip protocol rounds
CLUSTER_GOSSIP_FANOUTu83Number of peers to gossip with per round
CLUSTER_SUSPICION_MULTu84Multiplier for suspicion timeout (suspicion_mult * +gossip_interval)
CLUSTER_PROBE_INTERVAL_MSu642000Milliseconds between SWIM probe pings
CLUSTER_PROBE_TIMEOUT_MSu64500Timeout for a direct probe response
CLUSTER_INDIRECT_PROBESu83Number of indirect probes before suspicion
CLUSTER_TASK_TIMEOUT_SECSu64300Timeout for a routed task to complete
CLUSTER_MAX_NODESu1664Maximum cluster size
CLUSTER_TLS_CERTpath""Optional TLS certificate for cross-datacenter transport
CLUSTER_TLS_KEYpath""Optional TLS private key
CLUSTER_ADVERTISE_ADDRstringautoAddress advertised to other nodes (for NAT traversal)
CLUSTER_ADVERTISE_PORTu16same as bindPort advertised to other nodes
CLUSTER_REGIONstring""Logical region tag for locality-aware routing
CLUSTER_CAPABILITIESstring""Comma-separated capability tags (e.g., +gpu,high-memory,docker)
+

Minimal Configuration

+
CLUSTER_ENABLED=true
+CLUSTER_SECRET=my-very-long-pre-shared-key-at-least-32-chars
+

Cross-Datacenter +Configuration

+
CLUSTER_ENABLED=true
+CLUSTER_SECRET=my-very-long-pre-shared-key-at-least-32-chars
+CLUSTER_SEEDS=dc1-node1.example.com:9400,dc2-node1.example.com:9400
+CLUSTER_ADVERTISE_ADDR=203.0.113.10
+CLUSTER_REGION=us-east-1
+CLUSTER_TLS_CERT=/etc/optimclaw/cluster.crt
+CLUSTER_TLS_KEY=/etc/optimclaw/cluster.key
+

Discovery Protocol

+

Nodes discover each other using a UDP beacon protocol. When a node +starts with CLUSTER_ENABLED=true, it begins broadcasting +beacon packets on the configured broadcast address and port.

+

Beacon Packet Format

+
Offset  Size    Field
+0       4       Magic bytes: 0x4F 0x43 0x4D 0x53 ("OCMS")
+4       1       Protocol version (currently 0x01)
+5       2       Beacon port (big-endian u16)
+7       2       Transport port (big-endian u16)
+9       32      Node ID (UTF-8, zero-padded)
+41      32      HMAC-SHA256 of bytes 0..41 using CLUSTER_SECRET
+

Total beacon size: 73 bytes.

+

Discovery Sequence

+
    +
  1. On startup, the node broadcasts a beacon every +CLUSTER_BEACON_INTERVAL_MS milliseconds to +CLUSTER_BEACON_SUBNET:CLUSTER_BEACON_PORT.
    2. +
  2. All listening nodes receive the beacon, verify the HMAC against +their own CLUSTER_SECRET, and extract the sender’s +transport address.
    3. +
  3. If the beacon is from an unknown node, the receiving node initiates +a post-quantum key exchange (see below) over TCP to the sender’s +transport address.
    4. +
  4. Once the encrypted channel is established, the new node is added to +the membership list and the gossip protocol takes over.
    5. +
  5. In non-broadcast environments (cloud, cross-datacenter), set +CLUSTER_SEEDS to bootstrap. The node will contact seed +addresses directly instead of relying on broadcast.
    6. +
+

Beacons continue to be sent after joining to help new nodes discover +the cluster.

+

Post-Quantum Cryptography

+

All inter-node communication is encrypted using a hybrid post-quantum +scheme to protect against both classical and quantum adversaries.

+

Key Exchange: ML-KEM-768

+

ML-KEM-768 (formerly CRYSTALS-Kyber) is a lattice-based key +encapsulation mechanism standardized in FIPS 203. It provides IND-CCA2 +security at NIST security level 3 (roughly equivalent to AES-192).

+

The handshake proceeds as follows:

+
    +
  1. Initiator generates an ML-KEM-768 keypair +(ephemeral) and sends the public key (1184 bytes) along with its node ID +and a challenge derived from CLUSTER_SECRET.
    2. +
  2. Responder verifies the challenge, encapsulates a +shared secret using the received public key, and sends back the +ciphertext (1088 bytes) along with its own challenge response.
    3. +
  3. Both sides derive the same 256-bit shared secret from the ML-KEM +decapsulation.
    4. +
  4. The shared secret is combined with CLUSTER_SECRET via +HKDF-SHA256 to produce the final session key, binding the session to the +cluster identity.
    5. +
+

Authenticated Encryption: +AES-256-GCM

+

All messages after the handshake are encrypted with AES-256-GCM using +the derived session key:

+
    +
  • Nonce: 96-bit, incremented per message (with +sender-direction bit to avoid reuse)
    • +
  • AAD (Additional Authenticated Data): message type + +sequence number + sender node ID
    • +
  • Tag: 128-bit authentication tag appended to +ciphertext
    • +
+

Key Rotation

+

Session keys are rotated every 1 hour or after 2^32 messages, +whichever comes first. Rotation uses a new ML-KEM-768 encapsulation +within the existing encrypted channel.

+

Why Post-Quantum?

+

Mesh clusters may carry sensitive task data (credentials, personal +information, tool outputs). Harvest-now-decrypt-later attacks make it +prudent to deploy post-quantum cryptography today, even before +large-scale quantum computers exist.

+

Gossip Protocol (SWIM +Membership)

+

The cluster uses the SWIM (Scalable Weakly-consistent Infection-style +process group Membership) protocol for membership management and failure +detection.

+

Membership States

+

Each node maintains a membership list where every entry is in one of +three states:

+ + + + + + + + + + + + + + + + + + + + + +
StateMeaning
AliveNode is healthy and responsive
SuspectNode failed to respond to probes; may be down
DeadNode confirmed unreachable; removed from routing
+

Protocol Rounds

+

Every CLUSTER_GOSSIP_INTERVAL_MS, each node +performs:

+
    +
  1. Probe – Select a random alive member and send a +direct ping. If no ack within CLUSTER_PROBE_TIMEOUT_MS, +send indirect pings through CLUSTER_INDIRECT_PROBES random +members. If still no ack, mark the target as Suspect.
    2. +
  2. Gossip – Piggyback membership updates (state +changes, join/leave events) on probe messages. Each update includes a +Lamport timestamp for crdt-style conflict resolution.
    3. +
  3. Suspicion – Suspect nodes have +CLUSTER_SUSPICION_MULT * CLUSTER_GOSSIP_INTERVAL_MS to +refute by sending an Alive message with a higher incarnation number. If +not refuted, the node transitions to Dead.
    4. +
+

Consistency

+

SWIM provides eventual consistency. After a state change, all nodes +converge within O(log N) gossip rounds, where N is the cluster size. +With default settings (1s gossip interval, fanout 3), a 64-node cluster +converges in under 7 seconds.

+

Join and Leave

+
    +
  • Join: Triggered by beacon discovery or seed +contact. The joining node sends a Join message; existing members +propagate the new membership via gossip.
    • +
  • Graceful leave: A node sends a Leave message before +shutting down. Other nodes immediately mark it Dead without +suspicion.
    • +
  • Crash: Detected by the probe/suspicion mechanism +described above.
    • +
+

Task Routing Algorithm

+

When a task arrives at any node, the router decides whether to +execute it locally or forward it to a better-suited node. The decision +is based on a scoring formula applied to each alive node.

+

Scoring Formula

+
score(node) = w_load * (1 - load_ratio)
+            + w_latency * (1 - latency_ratio)
+            + w_capability * capability_match
+            + w_affinity * affinity_bonus
+            + w_locality * locality_bonus
+

Where:

+ +++++ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FactorWeight (default)Description
load_ratiow_load = 0.35Current jobs / max parallel jobs (lower is better)
latency_ratiow_latency = 0.25P95 RTT to this node / max observed RTT (lower is better)
capability_matchw_capability = 0.251.0 if node has all required capabilities, 0.0 otherwise
affinity_bonusw_affinity = 0.101.0 if the task has session affinity to this node, 0.0 +otherwise
locality_bonusw_locality = 0.051.0 if same CLUSTER_REGION, 0.5 if no region set, 0.0 +otherwise
+

Routing Decision

+
    +
  1. Compute score(node) for all alive nodes including +self.
    2. +
  2. If the local node’s score is within 10% of the best score, execute +locally (avoids unnecessary forwarding overhead).
    3. +
  3. Otherwise, forward the task to the highest-scoring node over the +encrypted channel.
    4. +
  4. If the target node fails to accept within 5 seconds, fall back to +local execution.
    5. +
  5. Results are returned to the originating node and delivered to the +original client.
    6. +
+

Session Affinity

+

Tasks that reference an ongoing conversation or job context are +preferentially routed to the node that holds that context. This avoids +expensive context transfer between nodes.

+

API Endpoints

+

The mesh cluster exposes monitoring endpoints on the standard web +gateway.

+

GET /api/mesh/status

+

Returns the cluster status for the local node.

+

Response:

+
{
+  "cluster_enabled": true,
+  "node_id": "node-alpha",
+  "state": "alive",
+  "region": "us-east-1",
+  "capabilities": ["gpu", "docker"],
+  "uptime_secs": 86423,
+  "transport": {
+    "bind_addr": "0.0.0.0:9400",
+    "advertise_addr": "203.0.113.10:9400",
+    "encryption": "ML-KEM-768 + AES-256-GCM",
+    "protocol_version": 1
+  },
+  "membership": {
+    "alive": 4,
+    "suspect": 0,
+    "dead": 1,
+    "total_seen": 5
+  },
+  "routing": {
+    "local_load": 0.35,
+    "tasks_routed_out": 142,
+    "tasks_routed_in": 87,
+    "tasks_failed_over": 3
+  }
+}
+

GET /api/mesh/nodes

+

Returns the membership list with per-node details.

+

Response:

+
{
+  "nodes": [
+    {
+      "node_id": "node-alpha",
+      "state": "alive",
+      "addr": "203.0.113.10:9400",
+      "region": "us-east-1",
+      "capabilities": ["gpu", "docker"],
+      "load_ratio": 0.35,
+      "latency_ms": 0,
+      "last_seen": "2026-03-29T12:34:56Z",
+      "incarnation": 7,
+      "is_self": true
+    },
+    {
+      "node_id": "node-beta",
+      "state": "alive",
+      "addr": "203.0.113.11:9400",
+      "region": "us-east-1",
+      "capabilities": ["high-memory"],
+      "load_ratio": 0.12,
+      "latency_ms": 2,
+      "last_seen": "2026-03-29T12:34:55Z",
+      "incarnation": 3,
+      "is_self": false
+    }
+  ]
+}
+

POST +/api/mesh/nodes/{node_id}/drain

+

Puts a node into drain mode (stops accepting new routed tasks, +finishes existing ones). Useful before maintenance.

+

Response:

+
{
+  "node_id": "node-beta",
+  "drained": true,
+  "remaining_tasks": 2
+}
+

Quick Start

+

Running Two Nodes on the +Same Machine

+

Terminal 1 (Node A):

+
export CLUSTER_ENABLED=true
+export CLUSTER_SECRET="change-me-to-a-strong-shared-secret-at-least-32-characters"
+export CLUSTER_NODE_ID=node-a
+export CLUSTER_BIND_PORT=9400
+export CLUSTER_BEACON_PORT=9401
+export DATABASE_URL=postgres://localhost/optimclaw_a
+
+optimclaw onboard   # if not already configured
+cargo run
+

Terminal 2 (Node B):

+
export CLUSTER_ENABLED=true
+export CLUSTER_SECRET="change-me-to-a-strong-shared-secret-at-least-32-characters"
+export CLUSTER_NODE_ID=node-b
+export CLUSTER_BIND_PORT=9410
+export CLUSTER_BEACON_PORT=9401   # same beacon port so they discover each other
+export DATABASE_URL=postgres://localhost/optimclaw_b
+
+optimclaw onboard
+cargo run
+

Within 5 seconds, both nodes should discover each other via UDP +beacons. Verify by hitting the status endpoint:

+
curl http://localhost:3000/api/mesh/status | jq .membership
+# {"alive": 2, "suspect": 0, "dead": 0, "total_seen": 2}
+

Running Across Machines

+

On each machine, set the same CLUSTER_SECRET and +either:

+
    +
  • Ensure UDP broadcast works on the local network (same subnet), +or
    • +
  • Set CLUSTER_SEEDS to the address of at least one other +node:
    • +
+
export CLUSTER_SEEDS=192.168.1.100:9400
+

Security Model

+

Threat Model

+

The mesh cluster is designed to be secure against:

+
    +
  1. Passive eavesdropping – All traffic is encrypted +with AES-256-GCM.
    2. +
  2. Active MITM – The ML-KEM-768 handshake is bound to +CLUSTER_SECRET, preventing interception by parties without +the pre-shared key.
    3. +
  3. Quantum adversaries – ML-KEM-768 provides +post-quantum security for key exchange.
    4. +
  4. Rogue node injection – Beacons are authenticated +with HMAC-SHA256; the handshake requires +CLUSTER_SECRET.
    5. +
  5. Replay attacks – Nonces are strictly monotonic; +replayed messages are rejected.
    6. +
  6. Partition exploitation – Nodes degrade to +independent operation; no split-brain data corruption.
    7. +
+

Trust Boundaries

+
    +
  • All nodes sharing the same CLUSTER_SECRET are in the +same trust domain.
    • +
  • A compromised CLUSTER_SECRET means any attacker can +join the cluster. Rotate the secret and restart all nodes if a +compromise is suspected.
    • +
  • Task data (including tool outputs) is encrypted in transit but +available in plaintext to any node in the cluster. Do not add untrusted +machines to a cluster that handles sensitive data.
    • +
+

Network Recommendations

+ ++++ + + + + + + + + + + + + + + + + + + + + +
DeploymentRecommendation
Same LANUDP beacons work out of the box. Use a firewall to restrict beacon +and transport ports to trusted hosts.
Cross-datacenterUse CLUSTER_SEEDS, disable beacons by setting +CLUSTER_BEACON_INTERVAL_MS=0, enable TLS +(CLUSTER_TLS_CERT / CLUSTER_TLS_KEY), and +restrict access via network ACLs.
Cloud (AWS/GCP/Azure)Use private VPC networking. Set CLUSTER_ADVERTISE_ADDR +to the private IP. Use security groups to restrict ports 9400-9401.
+

Troubleshooting

+

Nodes not discovering each +other

+
    +
  1. Check CLUSTER_SECRET – Must be +identical on all nodes. Even trailing whitespace matters.
    2. +
  2. Check beacon port – All nodes must use the same +CLUSTER_BEACON_PORT.
    3. +
  3. Check UDP broadcast – Some cloud providers and +corporate networks block UDP broadcast. Use CLUSTER_SEEDS +instead.
    4. +
  4. Check firewall – Ports +CLUSTER_BEACON_PORT (UDP) and +CLUSTER_BIND_PORT (TCP) must be open.
    5. +
  5. Check logs – Run with +RUST_LOG=optimclaw::cluster=debug to see beacon +send/receive events.
    6. +
+

Node stuck in Suspect state

+
    +
  • This typically means the node is slow to respond to probes.
    • +
  • Increase CLUSTER_PROBE_TIMEOUT_MS on busy nodes.
    • +
  • Increase CLUSTER_SUSPICION_MULT to give more time +before declaring a node dead.
    • +
  • Check if the node is CPU-starved or under heavy I/O load.
    • +
+

High task routing latency

+
    +
  • Check +curl localhost:3000/api/mesh/nodes | jq '.nodes[].latency_ms' +to identify slow links.
    • +
  • Tasks are only routed away from the local node if a remote node +scores >10% better. If most tasks should stay local, this is expected +behavior.
    • +
  • For cross-datacenter deployments, set CLUSTER_REGION on +each node so the locality bonus keeps tasks close.
    • +
+

Session key negotiation +failures

+
    +
  • Both nodes must support the same protocol version. Ensure all nodes +are running the same OptimClaw release.
    • +
  • If using TLS (CLUSTER_TLS_CERT), verify the certificate +is valid and trusted by the other node.
    • +
  • Check for clock skew greater than 5 minutes between nodes.
    • +
+

Node rejoining after +network partition

+
    +
  • After a partition heals, the previously-dead node sends beacons +again and is rediscovered.
    • +
  • The rejoining node increments its incarnation number to override the +Dead state in other nodes’ membership lists.
    • +
  • Any tasks that were in-flight to the partitioned node will have +timed out and been retried locally.
    • +
+

Diagnostic Commands

+
# Check cluster status
+curl -s http://localhost:3000/api/mesh/status | jq .
+
+# List all known nodes
+curl -s http://localhost:3000/api/mesh/nodes | jq .
+
+# Drain a node before maintenance
+curl -s -X POST http://localhost:3000/api/mesh/nodes/node-beta/drain | jq .
+
+# Watch cluster events in real time
+RUST_LOG=optimclaw::cluster=debug cargo run 2>&1 | grep cluster
+ + diff --git a/docs/pq-handshake.svg b/docs/pq-handshake.svg new file mode 100644 index 00000000..952f818d --- /dev/null +++ b/docs/pq-handshake.svg @@ -0,0 +1,74 @@ + + + + + + + + + + + Post-Quantum Handshake Protocol + ML-KEM-768 (Kyber) Key Encapsulation + AES-256-GCM Channel Encryption + + + + Node A (init) + + + + + Node B (resp) + + + + + + Generate ML-KEM-768 + keypair (pk, sk) + + + + + + + {node_id, kem_pk, sign_pk} + WebSocket message 1 + + + + + Encapsulate with A's pk: + (shared_secret, ct) = + KEM.Encaps(pk_A) + + + + + + + {node_id, ciphertext, sign_pk, ed25519_sig} + WebSocket message 2 + + + + + Verify sig, then decapsulate: + shared_secret = + KEM.Decaps(ct, sk_A) + + + + + HKDF-SHA256 → AES keys + (send_key, recv_key) + + + HKDF-SHA256 → AES keys + (send_key, recv_key) + + + + AES-256-GCM Encrypted Channel Active + nonce(8B) || ciphertext per frame | forward-secure + diff --git a/docs/task-routing.svg b/docs/task-routing.svg new file mode 100644 index 00000000..9f5b9a1c --- /dev/null +++ b/docs/task-routing.svg @@ -0,0 +1,56 @@ + + + + + + + + + + + Task Routing Algorithm + Intelligent capability-based task placement across the mesh + + + + + Task Arrives + + + + + + Hard Filters (Pass/Fail) + Required model? Required tools? Min VRAM? + Nodes failing any filter are excluded + + + + + + Soft Scoring (0.0 - 1.0) + + + 35%(1.0 - load)prefer idle nodes + 25%(vram_free / vram_total)prefer GPU headroom + 15%(mem_free / mem_total)prefer RAM headroom + 15%(1.0 / hop_count)prefer closer nodes + 10%(model_match)prefer exact match + + + + + + + Select Highest Score + + + + + + + Execute Locally + + + Route to Remote Peer +