mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-25 14:53:34 +00:00
docs: mesh cluster documentation, SVG diagrams, and ironclaw->optimclaw rename
- MESH_CLUSTER.md: full documentation for autonomous AI mesh network - LAZY_TOOLS.md: lazy tool loading for smaller LLMs - mesh-architecture.svg: colorful network topology diagram - task-routing.svg: scoring algorithm visualization - pq-handshake.svg: ML-KEM-768 key exchange sequence diagram - HTML docs with dark theme styling - Renamed ironclaw references to optimclaw in .env.example and README Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]>
This commit is contained in:
dingo
+15
-15
@@ -1,5 +1,5 @@
|
|||||||
# Database Configuration
|
# Database Configuration
|
||||||
DATABASE_URL=postgres://localhost/ironclaw
|
DATABASE_URL=postgres://localhost/optimclaw
|
||||||
DATABASE_POOL_SIZE=10
|
DATABASE_POOL_SIZE=10
|
||||||
|
|
||||||
# LLM Provider
|
# LLM Provider
|
||||||
@@ -26,19 +26,19 @@ DATABASE_POOL_SIZE=10
|
|||||||
|
|
||||||
# === GitHub Copilot ===
|
# === GitHub Copilot ===
|
||||||
# Uses the OAuth token from your Copilot IDE sign-in (for example
|
# Uses the OAuth token from your Copilot IDE sign-in (for example
|
||||||
# ~/.config/github-copilot/apps.json on Linux/macOS), or run `ironclaw onboard`
|
# ~/.config/github-copilot/apps.json on Linux/macOS), or run `optimclaw onboard`
|
||||||
# and choose the GitHub device login flow.
|
# and choose the GitHub device login flow.
|
||||||
# LLM_BACKEND=github_copilot
|
# LLM_BACKEND=github_copilot
|
||||||
# GITHUB_COPILOT_TOKEN=gho_...
|
# GITHUB_COPILOT_TOKEN=gho_...
|
||||||
# GITHUB_COPILOT_MODEL=gpt-4o
|
# GITHUB_COPILOT_MODEL=gpt-4o
|
||||||
# IronClaw injects standard VS Code Copilot headers automatically.
|
# OptimClaw injects standard VS Code Copilot headers automatically.
|
||||||
# Optional advanced headers for custom overrides:
|
# Optional advanced headers for custom overrides:
|
||||||
# GITHUB_COPILOT_EXTRA_HEADERS=Copilot-Integration-Id:vscode-chat
|
# GITHUB_COPILOT_EXTRA_HEADERS=Copilot-Integration-Id:vscode-chat
|
||||||
|
|
||||||
# === NEAR AI (Chat Completions API) ===
|
# === NEAR AI (Chat Completions API) ===
|
||||||
# Two auth modes:
|
# Two auth modes:
|
||||||
# 1. Session token (default): Uses browser OAuth (GitHub/Google) on first run.
|
# 1. Session token (default): Uses browser OAuth (GitHub/Google) on first run.
|
||||||
# Session token stored in ~/.ironclaw/session.json automatically.
|
# Session token stored in ~/.optimclaw/session.json automatically.
|
||||||
# Base URL defaults to https://private.near.ai
|
# Base URL defaults to https://private.near.ai
|
||||||
# 2. API key: Set NEARAI_API_KEY to use API key auth from cloud.near.ai.
|
# 2. API key: Set NEARAI_API_KEY to use API key auth from cloud.near.ai.
|
||||||
# Base URL defaults to https://cloud-api.near.ai
|
# Base URL defaults to https://cloud-api.near.ai
|
||||||
@@ -46,7 +46,7 @@ NEARAI_MODEL=Qwen/Qwen3.5-122B-A10B
|
|||||||
NEARAI_BASE_URL=https://private.near.ai
|
NEARAI_BASE_URL=https://private.near.ai
|
||||||
NEARAI_AUTH_URL=https://private.near.ai
|
NEARAI_AUTH_URL=https://private.near.ai
|
||||||
# NEARAI_SESSION_TOKEN=sess_... # hosting providers: set this
|
# NEARAI_SESSION_TOKEN=sess_... # hosting providers: set this
|
||||||
# NEARAI_SESSION_PATH=~/.ironclaw/session.json # optional, default shown
|
# NEARAI_SESSION_PATH=~/.optimclaw/session.json # optional, default shown
|
||||||
# NEARAI_API_KEY=... # API key from cloud.near.ai
|
# NEARAI_API_KEY=... # API key from cloud.near.ai
|
||||||
|
|
||||||
# Local LLM Providers (Ollama, LM Studio, vLLM, LiteLLM)
|
# Local LLM Providers (Ollama, LM Studio, vLLM, LiteLLM)
|
||||||
@@ -63,7 +63,7 @@ NEARAI_AUTH_URL=https://private.near.ai
|
|||||||
# LLM_API_KEY=sk-... # optional for local servers
|
# LLM_API_KEY=sk-... # optional for local servers
|
||||||
# Custom HTTP headers for OpenAI-compatible providers
|
# Custom HTTP headers for OpenAI-compatible providers
|
||||||
# Format: comma-separated key:value pairs
|
# Format: comma-separated key:value pairs
|
||||||
# LLM_EXTRA_HEADERS=HTTP-Referer:https://github.com/nearai/ironclaw,X-Title:ironclaw
|
# LLM_EXTRA_HEADERS=HTTP-Referer:https://github.com/nearai/optimclaw,X-Title:optimclaw
|
||||||
|
|
||||||
# === OpenRouter (300+ models via OpenAI-compatible) ===
|
# === OpenRouter (300+ models via OpenAI-compatible) ===
|
||||||
# LLM_MODEL=anthropic/claude-sonnet-4 # see openrouter.ai/models for IDs
|
# LLM_MODEL=anthropic/claude-sonnet-4 # see openrouter.ai/models for IDs
|
||||||
@@ -145,7 +145,7 @@ HTTP_HOST=0.0.0.0
|
|||||||
HTTP_PORT=8080
|
HTTP_PORT=8080
|
||||||
HTTP_WEBHOOK_SECRET=your-webhook-secret
|
HTTP_WEBHOOK_SECRET=your-webhook-secret
|
||||||
# Webhook authentication uses HMAC-SHA256 signature verification.
|
# Webhook authentication uses HMAC-SHA256 signature verification.
|
||||||
# Callers must send an X-IronClaw-Signature header with format: sha256=<hex_digest>
|
# Callers must send an X-OptimClaw-Signature header with format: sha256=<hex_digest>
|
||||||
# where the digest is HMAC-SHA256(HTTP_WEBHOOK_SECRET, raw_request_body) in lowercase hex.
|
# where the digest is HMAC-SHA256(HTTP_WEBHOOK_SECRET, raw_request_body) in lowercase hex.
|
||||||
#
|
#
|
||||||
# Example (bash):
|
# Example (bash):
|
||||||
@@ -153,7 +153,7 @@ HTTP_WEBHOOK_SECRET=your-webhook-secret
|
|||||||
# SIG=$(echo -n "$BODY" | openssl dgst -sha256 -hmac "$HTTP_WEBHOOK_SECRET" | cut -d' ' -f2)
|
# SIG=$(echo -n "$BODY" | openssl dgst -sha256 -hmac "$HTTP_WEBHOOK_SECRET" | cut -d' ' -f2)
|
||||||
# curl -X POST http://localhost:8080/webhook \
|
# curl -X POST http://localhost:8080/webhook \
|
||||||
# -H "Content-Type: application/json" \
|
# -H "Content-Type: application/json" \
|
||||||
# -H "X-IronClaw-Signature: sha256=$SIG" \
|
# -H "X-OptimClaw-Signature: sha256=$SIG" \
|
||||||
# -d "$BODY"
|
# -d "$BODY"
|
||||||
#
|
#
|
||||||
# DEPRECATED: Passing "secret" in the JSON body still works but will be removed in a future release.
|
# DEPRECATED: Passing "secret" in the JSON body still works but will be removed in a future release.
|
||||||
@@ -170,7 +170,7 @@ HTTP_WEBHOOK_SECRET=your-webhook-secret
|
|||||||
# SIGNAL_IGNORE_STORIES=true
|
# SIGNAL_IGNORE_STORIES=true
|
||||||
|
|
||||||
# Agent Settings
|
# Agent Settings
|
||||||
AGENT_NAME=ironclaw
|
AGENT_NAME=optimclaw
|
||||||
AGENT_MAX_PARALLEL_JOBS=5
|
AGENT_MAX_PARALLEL_JOBS=5
|
||||||
AGENT_JOB_TIMEOUT_SECS=3600
|
AGENT_JOB_TIMEOUT_SECS=3600
|
||||||
AGENT_STUCK_THRESHOLD_SECS=300
|
AGENT_STUCK_THRESHOLD_SECS=300
|
||||||
@@ -205,7 +205,7 @@ HEARTBEAT_NOTIFY_USER=default
|
|||||||
# # commands directly on the host. Without this
|
# # commands directly on the host. Without this
|
||||||
# # set to "true", full_access is downgraded to
|
# # set to "true", full_access is downgraded to
|
||||||
# # workspace_write.
|
# # workspace_write.
|
||||||
# SANDBOX_IMAGE=ironclaw-worker:latest
|
# SANDBOX_IMAGE=optimclaw-worker:latest
|
||||||
# SANDBOX_TIMEOUT_SECS=120
|
# SANDBOX_TIMEOUT_SECS=120
|
||||||
# SANDBOX_MEMORY_LIMIT_MB=2048
|
# SANDBOX_MEMORY_LIMIT_MB=2048
|
||||||
|
|
||||||
@@ -214,11 +214,11 @@ SAFETY_MAX_OUTPUT_LENGTH=100000
|
|||||||
SAFETY_INJECTION_CHECK_ENABLED=true
|
SAFETY_INJECTION_CHECK_ENABLED=true
|
||||||
|
|
||||||
# Restart Feature (Docker containers only)
|
# Restart Feature (Docker containers only)
|
||||||
# Set IRONCLAW_IN_DOCKER=true in the container entrypoint to enable the restart feature.
|
# Set OPTIMCLAW_IN_DOCKER=true in the container entrypoint to enable the restart feature.
|
||||||
# Without this, the restart tool and /restart command will be disabled.
|
# Without this, the restart tool and /restart command will be disabled.
|
||||||
# IRONCLAW_IN_DOCKER=false
|
# OPTIMCLAW_IN_DOCKER=false
|
||||||
# IRONCLAW_RESTART_DELAY=5 # default wait before exit (seconds, range: 1-30)
|
# OPTIMCLAW_RESTART_DELAY=5 # default wait before exit (seconds, range: 1-30)
|
||||||
# IRONCLAW_MAX_FAILURES=10 # max consecutive failures before container exits
|
# OPTIMCLAW_MAX_FAILURES=10 # max consecutive failures before container exits
|
||||||
|
|
||||||
# Logging
|
# Logging
|
||||||
RUST_LOG=ironclaw=debug,tower_http=debug
|
RUST_LOG=optimclaw=debug,tower_http=debug
|
||||||
|
|||||||
@@ -27,6 +27,8 @@
|
|||||||
<p align="center">
|
<p align="center">
|
||||||
<a href="#philosophy">Philosophy</a> •
|
<a href="#philosophy">Philosophy</a> •
|
||||||
<a href="#features">Features</a> •
|
<a href="#features">Features</a> •
|
||||||
|
<a href="#mesh-cluster">Mesh Cluster</a> •
|
||||||
|
<a href="#lazy-tools">Lazy Tools</a> •
|
||||||
<a href="#installation">Installation</a> •
|
<a href="#installation">Installation</a> •
|
||||||
<a href="#configuration">Configuration</a> •
|
<a href="#configuration">Configuration</a> •
|
||||||
<a href="#security">Security</a> •
|
<a href="#security">Security</a> •
|
||||||
@@ -79,6 +81,50 @@ OptimClaw is the AI assistant you can actually trust with your personal and prof
|
|||||||
- **Workspace Filesystem** - Flexible path-based storage for notes, logs, and context
|
- **Workspace Filesystem** - Flexible path-based storage for notes, logs, and context
|
||||||
- **Identity Files** - Maintain consistent personality and preferences across sessions
|
- **Identity Files** - Maintain consistent personality and preferences across sessions
|
||||||
|
|
||||||
|
## Mesh Cluster
|
||||||
|
|
||||||
|
OptimClaw instances can form an **autonomous AI mesh network** where nodes discover each other automatically, coordinate via a gossip protocol, and route tasks intelligently across the cluster.
|
||||||
|
|
||||||
|
Key highlights:
|
||||||
|
|
||||||
|
- **Zero-config discovery** -- UDP beacon broadcast finds peers on the local network automatically
|
||||||
|
- **Post-quantum encryption** -- ML-KEM-768 key exchange with AES-256-GCM authenticated encryption protects all inter-node traffic against both classical and quantum adversaries
|
||||||
|
- **SWIM gossip membership** -- Reliable failure detection and cluster state convergence in O(log N) rounds
|
||||||
|
- **Intelligent task routing** -- A scoring algorithm balances load, latency, capability match, session affinity, and region locality to pick the best node for each task
|
||||||
|
- **Graceful degradation** -- Nodes operate independently if connectivity is lost; no split-brain data corruption
|
||||||
|
|
||||||
|
Quick start (two nodes on one machine):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Terminal 1
|
||||||
|
export CLUSTER_ENABLED=true CLUSTER_SECRET="your-32-char-secret-here-change-me" CLUSTER_NODE_ID=node-a
|
||||||
|
cargo run
|
||||||
|
|
||||||
|
# Terminal 2
|
||||||
|
export CLUSTER_ENABLED=true CLUSTER_SECRET="your-32-char-secret-here-change-me" CLUSTER_NODE_ID=node-b CLUSTER_BIND_PORT=9410
|
||||||
|
cargo run
|
||||||
|
```
|
||||||
|
|
||||||
|
Monitor via `GET /api/mesh/status` and `GET /api/mesh/nodes`.
|
||||||
|
|
||||||
|
See [docs/MESH_CLUSTER.md](docs/MESH_CLUSTER.md) for the full guide covering architecture, configuration reference, security model, and troubleshooting.
|
||||||
|
|
||||||
|
## Lazy Tools
|
||||||
|
|
||||||
|
Lazy tool loading reduces the system prompt from approximately 13,000 tokens to approximately 4,000 tokens by deferring tool schemas that are not immediately needed.
|
||||||
|
|
||||||
|
Enable it with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export OPTIMCLAW_LAZY_TOOLS=1
|
||||||
|
```
|
||||||
|
|
||||||
|
When enabled, 12 core tools (echo, time, json, http, web_fetch, file_read, file_write, shell, memory_search, memory_write, message, tool_info) are loaded eagerly. All other tools -- including MCP, WASM, and skill tools -- are listed by name only. The LLM calls `tool_info` to load the full schema for any additional tool on demand.
|
||||||
|
|
||||||
|
This is recommended for production deployments and cost-sensitive usage with expensive models.
|
||||||
|
|
||||||
|
See [docs/LAZY_TOOLS.md](docs/LAZY_TOOLS.md) for the full guide.
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
### Prerequisites
|
### Prerequisites
|
||||||
|
|||||||
@@ -0,0 +1,126 @@
|
|||||||
|
# OptimClaw Lazy Tool Loading
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
Lazy tool loading is an optimization that dramatically reduces the initial system prompt size sent to the LLM. Instead of injecting full JSON schemas for every available tool into each request, OptimClaw loads only a small core set of tools eagerly and defers the rest until the LLM requests them.
|
||||||
|
|
||||||
|
**Impact:** System prompt size drops from approximately 13,000 tokens to approximately 4,000 tokens -- a 70% reduction. This saves cost on every LLM call and leaves more of the context window available for conversation history and tool outputs.
|
||||||
|
|
||||||
|
## How to Enable
|
||||||
|
|
||||||
|
Set the environment variable:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export OPTIMCLAW_LAZY_TOOLS=1
|
||||||
|
```
|
||||||
|
|
||||||
|
Or add it to `~/.optimclaw/.env`:
|
||||||
|
|
||||||
|
```env
|
||||||
|
OPTIMCLAW_LAZY_TOOLS=1
|
||||||
|
```
|
||||||
|
|
||||||
|
To disable (default behavior -- all tools loaded eagerly):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export OPTIMCLAW_LAZY_TOOLS=0
|
||||||
|
# or simply unset it
|
||||||
|
unset OPTIMCLAW_LAZY_TOOLS
|
||||||
|
```
|
||||||
|
|
||||||
|
## Core Tools (Always Loaded)
|
||||||
|
|
||||||
|
When lazy loading is enabled, the following 12 core tools are always included in the system prompt. These are the tools the LLM needs most frequently and cover the essential interaction patterns:
|
||||||
|
|
||||||
|
| # | Tool | Purpose |
|
||||||
|
|---|------|---------|
|
||||||
|
| 1 | `echo` | Return text to the user |
|
||||||
|
| 2 | `time` | Get current date and time |
|
||||||
|
| 3 | `json` | Parse and query JSON data |
|
||||||
|
| 4 | `http` | Make HTTP requests to allowed endpoints |
|
||||||
|
| 5 | `web_fetch` | Fetch and extract content from web pages |
|
||||||
|
| 6 | `file_read` | Read files from the workspace |
|
||||||
|
| 7 | `file_write` | Write files to the workspace |
|
||||||
|
| 8 | `shell` | Execute shell commands in the sandbox |
|
||||||
|
| 9 | `memory_search` | Search persistent memory (hybrid FTS + vector) |
|
||||||
|
| 10 | `memory_write` | Write to persistent memory |
|
||||||
|
| 11 | `message` | Send messages to channels |
|
||||||
|
| 12 | `tool_info` | Discover and load additional tools on demand |
|
||||||
|
|
||||||
|
## Tool Discovery with tool_info
|
||||||
|
|
||||||
|
The `tool_info` tool is the mechanism by which the LLM discovers and loads deferred tools. When the LLM determines it needs a tool that is not in its current context, it calls `tool_info` to retrieve the full schema.
|
||||||
|
|
||||||
|
### How It Works
|
||||||
|
|
||||||
|
1. The system prompt includes a brief note listing the names of all available (but not yet loaded) tools.
|
||||||
|
2. When the LLM needs one of these tools, it calls `tool_info` with the tool name or a search query.
|
||||||
|
3. `tool_info` returns the full JSON schema (parameters, description, examples) for the matched tools.
|
||||||
|
4. The LLM can then call the newly loaded tool in subsequent turns.
|
||||||
|
|
||||||
|
### tool_info Parameters
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"name": "tool_info",
|
||||||
|
"parameters": {
|
||||||
|
"query": {
|
||||||
|
"type": "string",
|
||||||
|
"description": "Exact tool name or keyword search query"
|
||||||
|
},
|
||||||
|
"max_results": {
|
||||||
|
"type": "number",
|
||||||
|
"description": "Maximum tools to return (default: 5)"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Example Flow
|
||||||
|
|
||||||
|
**System prompt includes:**
|
||||||
|
> Additional tools available (use `tool_info` to load): `job_create`, `job_status`, `job_cancel`, `routine_create`, `routine_list`, `skill_search`, `skill_install`, `extension_install`, `secrets_set`, `secrets_get`, ...
|
||||||
|
|
||||||
|
**LLM decides it needs to create a background job:**
|
||||||
|
|
||||||
|
```
|
||||||
|
LLM -> tool_info(query="job_create")
|
||||||
|
|
||||||
|
tool_info returns:
|
||||||
|
{
|
||||||
|
"tools": [{
|
||||||
|
"name": "job_create",
|
||||||
|
"description": "Create a new background job with the given prompt and priority",
|
||||||
|
"parameters": {
|
||||||
|
"prompt": { "type": "string", "required": true },
|
||||||
|
"priority": { "type": "number", "default": 5 },
|
||||||
|
"timeout_secs": { "type": "number", "default": 300 }
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
|
||||||
|
LLM -> job_create(prompt="Summarize today's news", priority=3)
|
||||||
|
```
|
||||||
|
|
||||||
|
## When to Use Lazy Loading
|
||||||
|
|
||||||
|
| Scenario | Recommendation |
|
||||||
|
|----------|----------------|
|
||||||
|
| Production deployment with many tools/MCP servers | Enable -- significant token savings |
|
||||||
|
| Development and debugging | Disable -- easier to see all available tools |
|
||||||
|
| Cost-sensitive usage with expensive models | Enable -- reduces per-request cost |
|
||||||
|
| Clusters with heterogeneous tool sets | Enable -- each node may have different tools |
|
||||||
|
| Simple setups with few tools (<15 total) | Either -- minimal difference |
|
||||||
|
|
||||||
|
## Performance Characteristics
|
||||||
|
|
||||||
|
- **First request:** Faster, because the system prompt is smaller and the LLM processes fewer tokens.
|
||||||
|
- **Tool discovery round-trip:** When the LLM calls `tool_info`, it adds one extra turn before the actual tool call. In practice this is rare because the 12 core tools handle the majority of interactions.
|
||||||
|
- **Subsequent requests in the same session:** Tool schemas loaded via `tool_info` remain in the conversation context for the duration of the session, so discovery cost is paid at most once per tool per session.
|
||||||
|
|
||||||
|
## Interaction with Other Features
|
||||||
|
|
||||||
|
- **MCP tools:** MCP-connected tool schemas are also deferred when lazy loading is enabled. They appear in the "additional tools available" list and are loaded via `tool_info`.
|
||||||
|
- **WASM tools:** Same behavior as MCP tools -- deferred and discoverable.
|
||||||
|
- **Skills:** Skill tools (`skill_list`, `skill_search`, `skill_install`, `skill_remove`) are deferred. The skill system itself is unaffected.
|
||||||
|
- **Mesh cluster:** Lazy loading is a per-node setting. Different nodes in a cluster can have different settings.
|
||||||
@@ -0,0 +1,447 @@
|
|||||||
|
# OptimClaw Mesh Cluster
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
The OptimClaw Mesh Cluster enables multiple OptimClaw instances to form an autonomous AI mesh network. Each node in the cluster operates independently while collaborating on tasks, sharing workload, and providing fault tolerance. Nodes discover each other automatically via UDP beacons, authenticate using post-quantum cryptography, and coordinate through a gossip-based membership protocol.
|
||||||
|
|
||||||
|
Key capabilities:
|
||||||
|
|
||||||
|
- **Automatic discovery** -- zero-configuration node joining via UDP broadcast beacons
|
||||||
|
- **Post-quantum security** -- ML-KEM-768 key encapsulation with AES-256-GCM authenticated encryption
|
||||||
|
- **Gossip membership** -- SWIM protocol for reliable failure detection and cluster state convergence
|
||||||
|
- **Intelligent task routing** -- scoring algorithm that considers load, latency, capability, and affinity
|
||||||
|
- **Graceful degradation** -- nodes operate independently if connectivity is lost
|
||||||
|
|
||||||
|
## Architecture
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────────────┐
|
||||||
|
│ Mesh Cluster │
|
||||||
|
│ │
|
||||||
|
┌──────────────┐ │ ┌──────────┐ Gossip ┌──────────┐ │
|
||||||
|
│ Client │──────►│ │ Node A │◄────────────►│ Node B │ │
|
||||||
|
│ (any chan.) │ │ │ │ (SWIM) │ │ │
|
||||||
|
└──────────────┘ │ │ ┌──────┐ │ │ ┌──────┐ │ │
|
||||||
|
│ │ │Agent │ │ │ │Agent │ │ │
|
||||||
|
│ │ │ Loop │ │ │ │ Loop │ │ │
|
||||||
|
│ │ └──────┘ │ │ └──────┘ │ │
|
||||||
|
│ │ ┌──────┐ │ │ ┌──────┐ │ │
|
||||||
|
│ │ │Tools │ │ │ │Tools │ │ │
|
||||||
|
│ │ └──────┘ │ │ └──────┘ │ │
|
||||||
|
│ └─────┬────┘ └────┬─────┘ │
|
||||||
|
│ │ │ │
|
||||||
|
│ │ UDP Beacons │ │
|
||||||
|
│ │◄───────────────────────►│ │
|
||||||
|
│ │ │ │
|
||||||
|
│ │ Task Routing │ │
|
||||||
|
│ │◄───────────────────────►│ │
|
||||||
|
│ │ (ML-KEM-768 + │ │
|
||||||
|
│ │ AES-256-GCM) │ │
|
||||||
|
│ ┌─────┴────┐ ┌────┴─────┐ │
|
||||||
|
│ │ Node C │◄────────────►│ Node D │ │
|
||||||
|
│ └──────────┘ Gossip └──────────┘ │
|
||||||
|
│ │
|
||||||
|
└─────────────────────────────────────────────┘
|
||||||
|
|
||||||
|
Data flow:
|
||||||
|
1. UDP beacon broadcast → node discovery
|
||||||
|
2. ML-KEM-768 handshake → shared secret
|
||||||
|
3. AES-256-GCM encrypted channel established
|
||||||
|
4. SWIM gossip protocol → membership state
|
||||||
|
5. Task routing → best node selected via scoring
|
||||||
|
6. Encrypted task dispatch + result collection
|
||||||
|
```
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
All cluster settings are controlled via environment variables prefixed with `CLUSTER_`. They can be set in `~/.optimclaw/.env` or passed directly.
|
||||||
|
|
||||||
|
| Variable | Type | Default | Description |
|
||||||
|
|----------|------|---------|-------------|
|
||||||
|
| `CLUSTER_ENABLED` | bool | `false` | Enable mesh cluster mode |
|
||||||
|
| `CLUSTER_NODE_ID` | string | auto (hostname) | Unique identifier for this node |
|
||||||
|
| `CLUSTER_BIND_ADDR` | string | `0.0.0.0` | Address to bind the cluster transport |
|
||||||
|
| `CLUSTER_BIND_PORT` | u16 | `9400` | Port for the encrypted cluster transport |
|
||||||
|
| `CLUSTER_BEACON_PORT` | u16 | `9401` | UDP port for discovery beacons |
|
||||||
|
| `CLUSTER_BEACON_INTERVAL_MS` | u64 | `5000` | Milliseconds between beacon broadcasts |
|
||||||
|
| `CLUSTER_BEACON_SUBNET` | string | `255.255.255.255` | Broadcast address for beacons |
|
||||||
|
| `CLUSTER_SECRET` | string | *required* | Pre-shared key for cluster authentication (min 32 chars) |
|
||||||
|
| `CLUSTER_SEEDS` | string | `""` | Comma-separated seed node addresses (`host:port`) for non-broadcast environments |
|
||||||
|
| `CLUSTER_GOSSIP_INTERVAL_MS` | u64 | `1000` | Milliseconds between gossip protocol rounds |
|
||||||
|
| `CLUSTER_GOSSIP_FANOUT` | u8 | `3` | Number of peers to gossip with per round |
|
||||||
|
| `CLUSTER_SUSPICION_MULT` | u8 | `4` | Multiplier for suspicion timeout (suspicion_mult * gossip_interval) |
|
||||||
|
| `CLUSTER_PROBE_INTERVAL_MS` | u64 | `2000` | Milliseconds between SWIM probe pings |
|
||||||
|
| `CLUSTER_PROBE_TIMEOUT_MS` | u64 | `500` | Timeout for a direct probe response |
|
||||||
|
| `CLUSTER_INDIRECT_PROBES` | u8 | `3` | Number of indirect probes before suspicion |
|
||||||
|
| `CLUSTER_TASK_TIMEOUT_SECS` | u64 | `300` | Timeout for a routed task to complete |
|
||||||
|
| `CLUSTER_MAX_NODES` | u16 | `64` | Maximum cluster size |
|
||||||
|
| `CLUSTER_TLS_CERT` | path | `""` | Optional TLS certificate for cross-datacenter transport |
|
||||||
|
| `CLUSTER_TLS_KEY` | path | `""` | Optional TLS private key |
|
||||||
|
| `CLUSTER_ADVERTISE_ADDR` | string | auto | Address advertised to other nodes (for NAT traversal) |
|
||||||
|
| `CLUSTER_ADVERTISE_PORT` | u16 | same as bind | Port advertised to other nodes |
|
||||||
|
| `CLUSTER_REGION` | string | `""` | Logical region tag for locality-aware routing |
|
||||||
|
| `CLUSTER_CAPABILITIES` | string | `""` | Comma-separated capability tags (e.g., `gpu,high-memory,docker`) |
|
||||||
|
|
||||||
|
### Minimal Configuration
|
||||||
|
|
||||||
|
```env
|
||||||
|
CLUSTER_ENABLED=true
|
||||||
|
CLUSTER_SECRET=my-very-long-pre-shared-key-at-least-32-chars
|
||||||
|
```
|
||||||
|
|
||||||
|
### Cross-Datacenter Configuration
|
||||||
|
|
||||||
|
```env
|
||||||
|
CLUSTER_ENABLED=true
|
||||||
|
CLUSTER_SECRET=my-very-long-pre-shared-key-at-least-32-chars
|
||||||
|
CLUSTER_SEEDS=dc1-node1.example.com:9400,dc2-node1.example.com:9400
|
||||||
|
CLUSTER_ADVERTISE_ADDR=203.0.113.10
|
||||||
|
CLUSTER_REGION=us-east-1
|
||||||
|
CLUSTER_TLS_CERT=/etc/optimclaw/cluster.crt
|
||||||
|
CLUSTER_TLS_KEY=/etc/optimclaw/cluster.key
|
||||||
|
```
|
||||||
|
|
||||||
|
## Discovery Protocol
|
||||||
|
|
||||||
|
Nodes discover each other using a UDP beacon protocol. When a node starts with `CLUSTER_ENABLED=true`, it begins broadcasting beacon packets on the configured broadcast address and port.
|
||||||
|
|
||||||
|
### Beacon Packet Format
|
||||||
|
|
||||||
|
```
|
||||||
|
Offset Size Field
|
||||||
|
0 4 Magic bytes: 0x4F 0x43 0x4D 0x53 ("OCMS")
|
||||||
|
4 1 Protocol version (currently 0x01)
|
||||||
|
5 2 Beacon port (big-endian u16)
|
||||||
|
7 2 Transport port (big-endian u16)
|
||||||
|
9 32 Node ID (UTF-8, zero-padded)
|
||||||
|
41 32 HMAC-SHA256 of bytes 0..41 using CLUSTER_SECRET
|
||||||
|
```
|
||||||
|
|
||||||
|
Total beacon size: 73 bytes.
|
||||||
|
|
||||||
|
### Discovery Sequence
|
||||||
|
|
||||||
|
1. On startup, the node broadcasts a beacon every `CLUSTER_BEACON_INTERVAL_MS` milliseconds to `CLUSTER_BEACON_SUBNET:CLUSTER_BEACON_PORT`.
|
||||||
|
2. All listening nodes receive the beacon, verify the HMAC against their own `CLUSTER_SECRET`, and extract the sender's transport address.
|
||||||
|
3. If the beacon is from an unknown node, the receiving node initiates a post-quantum key exchange (see below) over TCP to the sender's transport address.
|
||||||
|
4. Once the encrypted channel is established, the new node is added to the membership list and the gossip protocol takes over.
|
||||||
|
5. In non-broadcast environments (cloud, cross-datacenter), set `CLUSTER_SEEDS` to bootstrap. The node will contact seed addresses directly instead of relying on broadcast.
|
||||||
|
|
||||||
|
Beacons continue to be sent after joining to help new nodes discover the cluster.
|
||||||
|
|
||||||
|
## Post-Quantum Cryptography
|
||||||
|
|
||||||
|
All inter-node communication is encrypted using a hybrid post-quantum scheme to protect against both classical and quantum adversaries.
|
||||||
|
|
||||||
|
### Key Exchange: ML-KEM-768
|
||||||
|
|
||||||
|
ML-KEM-768 (formerly CRYSTALS-Kyber) is a lattice-based key encapsulation mechanism standardized in FIPS 203. It provides IND-CCA2 security at NIST security level 3 (roughly equivalent to AES-192).
|
||||||
|
|
||||||
|
The handshake proceeds as follows:
|
||||||
|
|
||||||
|
1. **Initiator** generates an ML-KEM-768 keypair (ephemeral) and sends the public key (1184 bytes) along with its node ID and a challenge derived from `CLUSTER_SECRET`.
|
||||||
|
2. **Responder** verifies the challenge, encapsulates a shared secret using the received public key, and sends back the ciphertext (1088 bytes) along with its own challenge response.
|
||||||
|
3. Both sides derive the same 256-bit shared secret from the ML-KEM decapsulation.
|
||||||
|
4. The shared secret is combined with `CLUSTER_SECRET` via HKDF-SHA256 to produce the final session key, binding the session to the cluster identity.
|
||||||
|
|
||||||
|
### Authenticated Encryption: AES-256-GCM
|
||||||
|
|
||||||
|
All messages after the handshake are encrypted with AES-256-GCM using the derived session key:
|
||||||
|
|
||||||
|
- **Nonce**: 96-bit, incremented per message (with sender-direction bit to avoid reuse)
|
||||||
|
- **AAD (Additional Authenticated Data)**: message type + sequence number + sender node ID
|
||||||
|
- **Tag**: 128-bit authentication tag appended to ciphertext
|
||||||
|
|
||||||
|
### Key Rotation
|
||||||
|
|
||||||
|
Session keys are rotated every 1 hour or after 2^32 messages, whichever comes first. Rotation uses a new ML-KEM-768 encapsulation within the existing encrypted channel.
|
||||||
|
|
||||||
|
### Why Post-Quantum?
|
||||||
|
|
||||||
|
Mesh clusters may carry sensitive task data (credentials, personal information, tool outputs). Harvest-now-decrypt-later attacks make it prudent to deploy post-quantum cryptography today, even before large-scale quantum computers exist.
|
||||||
|
|
||||||
|
## Gossip Protocol (SWIM Membership)
|
||||||
|
|
||||||
|
The cluster uses the SWIM (Scalable Weakly-consistent Infection-style process group Membership) protocol for membership management and failure detection.
|
||||||
|
|
||||||
|
### Membership States
|
||||||
|
|
||||||
|
Each node maintains a membership list where every entry is in one of three states:
|
||||||
|
|
||||||
|
| State | Meaning |
|
||||||
|
|-------|---------|
|
||||||
|
| **Alive** | Node is healthy and responsive |
|
||||||
|
| **Suspect** | Node failed to respond to probes; may be down |
|
||||||
|
| **Dead** | Node confirmed unreachable; removed from routing |
|
||||||
|
|
||||||
|
### Protocol Rounds
|
||||||
|
|
||||||
|
Every `CLUSTER_GOSSIP_INTERVAL_MS`, each node performs:
|
||||||
|
|
||||||
|
1. **Probe** -- Select a random alive member and send a direct ping. If no ack within `CLUSTER_PROBE_TIMEOUT_MS`, send indirect pings through `CLUSTER_INDIRECT_PROBES` random members. If still no ack, mark the target as Suspect.
|
||||||
|
2. **Gossip** -- Piggyback membership updates (state changes, join/leave events) on probe messages. Each update includes a Lamport timestamp for crdt-style conflict resolution.
|
||||||
|
3. **Suspicion** -- Suspect nodes have `CLUSTER_SUSPICION_MULT * CLUSTER_GOSSIP_INTERVAL_MS` to refute by sending an Alive message with a higher incarnation number. If not refuted, the node transitions to Dead.
|
||||||
|
|
||||||
|
### Consistency
|
||||||
|
|
||||||
|
SWIM provides eventual consistency. After a state change, all nodes converge within O(log N) gossip rounds, where N is the cluster size. With default settings (1s gossip interval, fanout 3), a 64-node cluster converges in under 7 seconds.
|
||||||
|
|
||||||
|
### Join and Leave
|
||||||
|
|
||||||
|
- **Join**: Triggered by beacon discovery or seed contact. The joining node sends a Join message; existing members propagate the new membership via gossip.
|
||||||
|
- **Graceful leave**: A node sends a Leave message before shutting down. Other nodes immediately mark it Dead without suspicion.
|
||||||
|
- **Crash**: Detected by the probe/suspicion mechanism described above.
|
||||||
|
|
||||||
|
## Task Routing Algorithm
|
||||||
|
|
||||||
|
When a task arrives at any node, the router decides whether to execute it locally or forward it to a better-suited node. The decision is based on a scoring formula applied to each alive node.
|
||||||
|
|
||||||
|
### Scoring Formula
|
||||||
|
|
||||||
|
```
|
||||||
|
score(node) = w_load * (1 - load_ratio)
|
||||||
|
+ w_latency * (1 - latency_ratio)
|
||||||
|
+ w_capability * capability_match
|
||||||
|
+ w_affinity * affinity_bonus
|
||||||
|
+ w_locality * locality_bonus
|
||||||
|
```
|
||||||
|
|
||||||
|
Where:
|
||||||
|
|
||||||
|
| Factor | Weight (default) | Description |
|
||||||
|
|--------|-------------------|-------------|
|
||||||
|
| `load_ratio` | `w_load = 0.35` | Current jobs / max parallel jobs (lower is better) |
|
||||||
|
| `latency_ratio` | `w_latency = 0.25` | P95 RTT to this node / max observed RTT (lower is better) |
|
||||||
|
| `capability_match` | `w_capability = 0.25` | 1.0 if node has all required capabilities, 0.0 otherwise |
|
||||||
|
| `affinity_bonus` | `w_affinity = 0.10` | 1.0 if the task has session affinity to this node, 0.0 otherwise |
|
||||||
|
| `locality_bonus` | `w_locality = 0.05` | 1.0 if same `CLUSTER_REGION`, 0.5 if no region set, 0.0 otherwise |
|
||||||
|
|
||||||
|
### Routing Decision
|
||||||
|
|
||||||
|
1. Compute `score(node)` for all alive nodes including self.
|
||||||
|
2. If the local node's score is within 10% of the best score, execute locally (avoids unnecessary forwarding overhead).
|
||||||
|
3. Otherwise, forward the task to the highest-scoring node over the encrypted channel.
|
||||||
|
4. If the target node fails to accept within 5 seconds, fall back to local execution.
|
||||||
|
5. Results are returned to the originating node and delivered to the original client.
|
||||||
|
|
||||||
|
### Session Affinity
|
||||||
|
|
||||||
|
Tasks that reference an ongoing conversation or job context are preferentially routed to the node that holds that context. This avoids expensive context transfer between nodes.
|
||||||
|
|
||||||
|
## API Endpoints
|
||||||
|
|
||||||
|
The mesh cluster exposes monitoring endpoints on the standard web gateway.
|
||||||
|
|
||||||
|
### GET /api/mesh/status
|
||||||
|
|
||||||
|
Returns the cluster status for the local node.
|
||||||
|
|
||||||
|
**Response:**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"cluster_enabled": true,
|
||||||
|
"node_id": "node-alpha",
|
||||||
|
"state": "alive",
|
||||||
|
"region": "us-east-1",
|
||||||
|
"capabilities": ["gpu", "docker"],
|
||||||
|
"uptime_secs": 86423,
|
||||||
|
"transport": {
|
||||||
|
"bind_addr": "0.0.0.0:9400",
|
||||||
|
"advertise_addr": "203.0.113.10:9400",
|
||||||
|
"encryption": "ML-KEM-768 + AES-256-GCM",
|
||||||
|
"protocol_version": 1
|
||||||
|
},
|
||||||
|
"membership": {
|
||||||
|
"alive": 4,
|
||||||
|
"suspect": 0,
|
||||||
|
"dead": 1,
|
||||||
|
"total_seen": 5
|
||||||
|
},
|
||||||
|
"routing": {
|
||||||
|
"local_load": 0.35,
|
||||||
|
"tasks_routed_out": 142,
|
||||||
|
"tasks_routed_in": 87,
|
||||||
|
"tasks_failed_over": 3
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### GET /api/mesh/nodes
|
||||||
|
|
||||||
|
Returns the membership list with per-node details.
|
||||||
|
|
||||||
|
**Response:**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"nodes": [
|
||||||
|
{
|
||||||
|
"node_id": "node-alpha",
|
||||||
|
"state": "alive",
|
||||||
|
"addr": "203.0.113.10:9400",
|
||||||
|
"region": "us-east-1",
|
||||||
|
"capabilities": ["gpu", "docker"],
|
||||||
|
"load_ratio": 0.35,
|
||||||
|
"latency_ms": 0,
|
||||||
|
"last_seen": "2026-03-29T12:34:56Z",
|
||||||
|
"incarnation": 7,
|
||||||
|
"is_self": true
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"node_id": "node-beta",
|
||||||
|
"state": "alive",
|
||||||
|
"addr": "203.0.113.11:9400",
|
||||||
|
"region": "us-east-1",
|
||||||
|
"capabilities": ["high-memory"],
|
||||||
|
"load_ratio": 0.12,
|
||||||
|
"latency_ms": 2,
|
||||||
|
"last_seen": "2026-03-29T12:34:55Z",
|
||||||
|
"incarnation": 3,
|
||||||
|
"is_self": false
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### POST /api/mesh/nodes/{node_id}/drain
|
||||||
|
|
||||||
|
Puts a node into drain mode (stops accepting new routed tasks, finishes existing ones). Useful before maintenance.
|
||||||
|
|
||||||
|
**Response:**
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"node_id": "node-beta",
|
||||||
|
"drained": true,
|
||||||
|
"remaining_tasks": 2
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
## Quick Start
|
||||||
|
|
||||||
|
### Running Two Nodes on the Same Machine
|
||||||
|
|
||||||
|
**Terminal 1 (Node A):**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export CLUSTER_ENABLED=true
|
||||||
|
export CLUSTER_SECRET="change-me-to-a-strong-shared-secret-at-least-32-characters"
|
||||||
|
export CLUSTER_NODE_ID=node-a
|
||||||
|
export CLUSTER_BIND_PORT=9400
|
||||||
|
export CLUSTER_BEACON_PORT=9401
|
||||||
|
export DATABASE_URL=postgres://localhost/optimclaw_a
|
||||||
|
|
||||||
|
optimclaw onboard # if not already configured
|
||||||
|
cargo run
|
||||||
|
```
|
||||||
|
|
||||||
|
**Terminal 2 (Node B):**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export CLUSTER_ENABLED=true
|
||||||
|
export CLUSTER_SECRET="change-me-to-a-strong-shared-secret-at-least-32-characters"
|
||||||
|
export CLUSTER_NODE_ID=node-b
|
||||||
|
export CLUSTER_BIND_PORT=9410
|
||||||
|
export CLUSTER_BEACON_PORT=9401 # same beacon port so they discover each other
|
||||||
|
export DATABASE_URL=postgres://localhost/optimclaw_b
|
||||||
|
|
||||||
|
optimclaw onboard
|
||||||
|
cargo run
|
||||||
|
```
|
||||||
|
|
||||||
|
Within 5 seconds, both nodes should discover each other via UDP beacons. Verify by hitting the status endpoint:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
curl http://localhost:3000/api/mesh/status | jq .membership
|
||||||
|
# {"alive": 2, "suspect": 0, "dead": 0, "total_seen": 2}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Running Across Machines
|
||||||
|
|
||||||
|
On each machine, set the same `CLUSTER_SECRET` and either:
|
||||||
|
|
||||||
|
- Ensure UDP broadcast works on the local network (same subnet), or
|
||||||
|
- Set `CLUSTER_SEEDS` to the address of at least one other node:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
export CLUSTER_SEEDS=192.168.1.100:9400
|
||||||
|
```
|
||||||
|
|
||||||
|
## Security Model
|
||||||
|
|
||||||
|
### Threat Model
|
||||||
|
|
||||||
|
The mesh cluster is designed to be secure against:
|
||||||
|
|
||||||
|
1. **Passive eavesdropping** -- All traffic is encrypted with AES-256-GCM.
|
||||||
|
2. **Active MITM** -- The ML-KEM-768 handshake is bound to `CLUSTER_SECRET`, preventing interception by parties without the pre-shared key.
|
||||||
|
3. **Quantum adversaries** -- ML-KEM-768 provides post-quantum security for key exchange.
|
||||||
|
4. **Rogue node injection** -- Beacons are authenticated with HMAC-SHA256; the handshake requires `CLUSTER_SECRET`.
|
||||||
|
5. **Replay attacks** -- Nonces are strictly monotonic; replayed messages are rejected.
|
||||||
|
6. **Partition exploitation** -- Nodes degrade to independent operation; no split-brain data corruption.
|
||||||
|
|
||||||
|
### Trust Boundaries
|
||||||
|
|
||||||
|
- All nodes sharing the same `CLUSTER_SECRET` are in the same trust domain.
|
||||||
|
- A compromised `CLUSTER_SECRET` means any attacker can join the cluster. Rotate the secret and restart all nodes if a compromise is suspected.
|
||||||
|
- Task data (including tool outputs) is encrypted in transit but available in plaintext to any node in the cluster. Do not add untrusted machines to a cluster that handles sensitive data.
|
||||||
|
|
||||||
|
### Network Recommendations
|
||||||
|
|
||||||
|
| Deployment | Recommendation |
|
||||||
|
|------------|----------------|
|
||||||
|
| Same LAN | UDP beacons work out of the box. Use a firewall to restrict beacon and transport ports to trusted hosts. |
|
||||||
|
| Cross-datacenter | Use `CLUSTER_SEEDS`, disable beacons by setting `CLUSTER_BEACON_INTERVAL_MS=0`, enable TLS (`CLUSTER_TLS_CERT` / `CLUSTER_TLS_KEY`), and restrict access via network ACLs. |
|
||||||
|
| Cloud (AWS/GCP/Azure) | Use private VPC networking. Set `CLUSTER_ADVERTISE_ADDR` to the private IP. Use security groups to restrict ports 9400-9401. |
|
||||||
|
|
||||||
|
## Troubleshooting
|
||||||
|
|
||||||
|
### Nodes not discovering each other
|
||||||
|
|
||||||
|
1. **Check `CLUSTER_SECRET`** -- Must be identical on all nodes. Even trailing whitespace matters.
|
||||||
|
2. **Check beacon port** -- All nodes must use the same `CLUSTER_BEACON_PORT`.
|
||||||
|
3. **Check UDP broadcast** -- Some cloud providers and corporate networks block UDP broadcast. Use `CLUSTER_SEEDS` instead.
|
||||||
|
4. **Check firewall** -- Ports `CLUSTER_BEACON_PORT` (UDP) and `CLUSTER_BIND_PORT` (TCP) must be open.
|
||||||
|
5. **Check logs** -- Run with `RUST_LOG=optimclaw::cluster=debug` to see beacon send/receive events.
|
||||||
|
|
||||||
|
### Node stuck in Suspect state
|
||||||
|
|
||||||
|
- This typically means the node is slow to respond to probes.
|
||||||
|
- Increase `CLUSTER_PROBE_TIMEOUT_MS` on busy nodes.
|
||||||
|
- Increase `CLUSTER_SUSPICION_MULT` to give more time before declaring a node dead.
|
||||||
|
- Check if the node is CPU-starved or under heavy I/O load.
|
||||||
|
|
||||||
|
### High task routing latency
|
||||||
|
|
||||||
|
- Check `curl localhost:3000/api/mesh/nodes | jq '.nodes[].latency_ms'` to identify slow links.
|
||||||
|
- Tasks are only routed away from the local node if a remote node scores >10% better. If most tasks should stay local, this is expected behavior.
|
||||||
|
- For cross-datacenter deployments, set `CLUSTER_REGION` on each node so the locality bonus keeps tasks close.
|
||||||
|
|
||||||
|
### Session key negotiation failures
|
||||||
|
|
||||||
|
- Both nodes must support the same protocol version. Ensure all nodes are running the same OptimClaw release.
|
||||||
|
- If using TLS (`CLUSTER_TLS_CERT`), verify the certificate is valid and trusted by the other node.
|
||||||
|
- Check for clock skew greater than 5 minutes between nodes.
|
||||||
|
|
||||||
|
### Node rejoining after network partition
|
||||||
|
|
||||||
|
- After a partition heals, the previously-dead node sends beacons again and is rediscovered.
|
||||||
|
- The rejoining node increments its incarnation number to override the Dead state in other nodes' membership lists.
|
||||||
|
- Any tasks that were in-flight to the partitioned node will have timed out and been retried locally.
|
||||||
|
|
||||||
|
### Diagnostic Commands
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Check cluster status
|
||||||
|
curl -s http://localhost:3000/api/mesh/status | jq .
|
||||||
|
|
||||||
|
# List all known nodes
|
||||||
|
curl -s http://localhost:3000/api/mesh/nodes | jq .
|
||||||
|
|
||||||
|
# Drain a node before maintenance
|
||||||
|
curl -s -X POST http://localhost:3000/api/mesh/nodes/node-beta/drain | jq .
|
||||||
|
|
||||||
|
# Watch cluster events in real time
|
||||||
|
RUST_LOG=optimclaw::cluster=debug cargo run 2>&1 | grep cluster
|
||||||
|
```
|
||||||
@@ -0,0 +1,112 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 900 600" font-family="Inter, system-ui, sans-serif">
|
||||||
|
<defs>
|
||||||
|
<linearGradient id="bg" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||||
|
<stop offset="0%" style="stop-color:#1a1a2e"/>
|
||||||
|
<stop offset="100%" style="stop-color:#16213e"/>
|
||||||
|
</linearGradient>
|
||||||
|
<linearGradient id="node-grad" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||||
|
<stop offset="0%" style="stop-color:#0f3460"/>
|
||||||
|
<stop offset="100%" style="stop-color:#1a1a2e"/>
|
||||||
|
</linearGradient>
|
||||||
|
<filter id="glow">
|
||||||
|
<feGaussianBlur stdDeviation="3" result="blur"/>
|
||||||
|
<feMerge><feMergeNode in="blur"/><feMergeNode in="SourceGraphic"/></feMerge>
|
||||||
|
</filter>
|
||||||
|
<filter id="shadow">
|
||||||
|
<feDropShadow dx="2" dy="2" stdDeviation="4" flood-color="#000" flood-opacity="0.3"/>
|
||||||
|
</filter>
|
||||||
|
</defs>
|
||||||
|
|
||||||
|
<!-- Background -->
|
||||||
|
<rect width="900" height="600" fill="url(#bg)" rx="12"/>
|
||||||
|
|
||||||
|
<!-- Title -->
|
||||||
|
<text x="450" y="45" text-anchor="middle" fill="#e94560" font-size="24" font-weight="700" letter-spacing="1">OptimClaw Mesh Network</text>
|
||||||
|
<text x="450" y="68" text-anchor="middle" fill="#8899aa" font-size="12">Autonomous Peer-to-Peer AI Agent Cluster with Post-Quantum Encryption</text>
|
||||||
|
|
||||||
|
<!-- Connection lines (behind nodes) -->
|
||||||
|
<g stroke="#e94560" stroke-opacity="0.3" stroke-width="2" stroke-dasharray="6,4">
|
||||||
|
<!-- A-B --> <line x1="180" y1="200" x2="450" y2="160"/>
|
||||||
|
<!-- B-C --> <line x1="550" y1="160" x2="720" y2="200"/>
|
||||||
|
<!-- A-E --> <line x1="180" y1="280" x2="350" y2="380"/>
|
||||||
|
<!-- B-E --> <line x1="450" y1="240" x2="420" y2="380"/>
|
||||||
|
<!-- C-E --> <line x1="720" y1="280" x2="500" y2="380"/>
|
||||||
|
<!-- C-D --> <line x1="790" y1="280" x2="790" y2="380"/>
|
||||||
|
<!-- B-D --> <line x1="550" y1="240" x2="750" y2="380"/>
|
||||||
|
</g>
|
||||||
|
|
||||||
|
<!-- Encrypted channel indicators -->
|
||||||
|
<g fill="#00d4aa" font-size="9" font-weight="600">
|
||||||
|
<text x="310" y="175" text-anchor="middle" transform="rotate(-10,310,175)">ML-KEM-768</text>
|
||||||
|
<text x="640" y="175" text-anchor="middle" transform="rotate(10,640,175)">AES-256-GCM</text>
|
||||||
|
<text x="260" y="340" text-anchor="middle" transform="rotate(25,260,340)">PQ Encrypted</text>
|
||||||
|
<text x="650" y="340" text-anchor="middle" transform="rotate(25,650,340)">PQ Encrypted</text>
|
||||||
|
</g>
|
||||||
|
|
||||||
|
<!-- Node A: Laptop -->
|
||||||
|
<g filter="url(#shadow)">
|
||||||
|
<rect x="100" y="170" width="160" height="120" rx="10" fill="url(#node-grad)" stroke="#e94560" stroke-width="2"/>
|
||||||
|
<circle cx="130" cy="195" r="8" fill="#00d4aa"/>
|
||||||
|
<text x="145" y="200" fill="#fff" font-size="14" font-weight="600">Node A</text>
|
||||||
|
<text x="120" y="220" fill="#8899aa" font-size="11">Laptop | 6GB VRAM</text>
|
||||||
|
<text x="120" y="238" fill="#fab283" font-size="10">RTX 3060 Mobile</text>
|
||||||
|
<text x="120" y="255" fill="#5c9cf5" font-size="10">qwen2.5-coder:7b</text>
|
||||||
|
<text x="120" y="275" fill="#7fd88f" font-size="10">Load: 35% | 12 tools</text>
|
||||||
|
</g>
|
||||||
|
|
||||||
|
<!-- Node B: Server -->
|
||||||
|
<g filter="url(#shadow)">
|
||||||
|
<rect x="370" y="130" width="160" height="120" rx="10" fill="url(#node-grad)" stroke="#fab283" stroke-width="2"/>
|
||||||
|
<circle cx="400" cy="155" r="8" fill="#00d4aa"/>
|
||||||
|
<text x="415" y="160" fill="#fff" font-size="14" font-weight="600">Node B</text>
|
||||||
|
<text x="390" y="180" fill="#8899aa" font-size="11">Server | 24GB VRAM</text>
|
||||||
|
<text x="390" y="198" fill="#fab283" font-size="10">RTX 4090</text>
|
||||||
|
<text x="390" y="215" fill="#5c9cf5" font-size="10">qwen3-coder:30b</text>
|
||||||
|
<text x="390" y="235" fill="#7fd88f" font-size="10">Load: 10% | 30 tools</text>
|
||||||
|
</g>
|
||||||
|
|
||||||
|
<!-- Node C: Cloud -->
|
||||||
|
<g filter="url(#shadow)">
|
||||||
|
<rect x="640" y="170" width="160" height="120" rx="10" fill="url(#node-grad)" stroke="#5c9cf5" stroke-width="2"/>
|
||||||
|
<circle cx="670" cy="195" r="8" fill="#00d4aa"/>
|
||||||
|
<text x="685" y="200" fill="#fff" font-size="14" font-weight="600">Node C</text>
|
||||||
|
<text x="660" y="220" fill="#8899aa" font-size="11">Cloud | 80GB VRAM</text>
|
||||||
|
<text x="660" y="238" fill="#fab283" font-size="10">A100</text>
|
||||||
|
<text x="660" y="255" fill="#5c9cf5" font-size="10">llama3.1:70b</text>
|
||||||
|
<text x="660" y="275" fill="#7fd88f" font-size="10">Load: 5% | 30 tools</text>
|
||||||
|
</g>
|
||||||
|
|
||||||
|
<!-- Node D: Remote -->
|
||||||
|
<g filter="url(#shadow)">
|
||||||
|
<rect x="700" y="350" width="160" height="120" rx="10" fill="url(#node-grad)" stroke="#9d7cd8" stroke-width="2"/>
|
||||||
|
<circle cx="730" cy="375" r="8" fill="#f5a742"/>
|
||||||
|
<text x="745" y="380" fill="#fff" font-size="14" font-weight="600">Node D</text>
|
||||||
|
<text x="720" y="400" fill="#8899aa" font-size="11">Remote | 48GB VRAM</text>
|
||||||
|
<text x="720" y="418" fill="#fab283" font-size="10">2x RTX 3090</text>
|
||||||
|
<text x="720" y="435" fill="#5c9cf5" font-size="10">deepseek-coder:33b</text>
|
||||||
|
<text x="720" y="455" fill="#f5a742" font-size="10">Load: 72% | 30 tools</text>
|
||||||
|
</g>
|
||||||
|
|
||||||
|
<!-- Node E: Edge -->
|
||||||
|
<g filter="url(#shadow)">
|
||||||
|
<rect x="310" y="350" width="160" height="120" rx="10" fill="url(#node-grad)" stroke="#7fd88f" stroke-width="2"/>
|
||||||
|
<circle cx="340" cy="375" r="8" fill="#00d4aa"/>
|
||||||
|
<text x="355" y="380" fill="#fff" font-size="14" font-weight="600">Node E</text>
|
||||||
|
<text x="330" y="400" fill="#8899aa" font-size="11">Edge | 8GB VRAM</text>
|
||||||
|
<text x="330" y="418" fill="#fab283" font-size="10">RTX 4060</text>
|
||||||
|
<text x="330" y="435" fill="#5c9cf5" font-size="10">qwen3:8b</text>
|
||||||
|
<text x="330" y="455" fill="#7fd88f" font-size="10">Load: 20% | 12 tools</text>
|
||||||
|
</g>
|
||||||
|
|
||||||
|
<!-- Legend -->
|
||||||
|
<g transform="translate(50,510)">
|
||||||
|
<rect width="800" height="70" rx="8" fill="#0f3460" fill-opacity="0.5" stroke="#333" stroke-width="1"/>
|
||||||
|
<text x="20" y="22" fill="#fff" font-size="12" font-weight="600">Protocol Stack</text>
|
||||||
|
<g transform="translate(20,35)">
|
||||||
|
<rect width="12" height="12" rx="2" fill="#e94560"/><text x="18" y="11" fill="#ccc" font-size="10">UDP Beacon Discovery (port 9900)</text>
|
||||||
|
<rect x="200" width="12" height="12" rx="2" fill="#00d4aa"/><text x="218" y="11" fill="#ccc" font-size="10">ML-KEM-768 Key Exchange</text>
|
||||||
|
<rect x="400" width="12" height="12" rx="2" fill="#fab283"/><text x="418" y="11" fill="#ccc" font-size="10">AES-256-GCM Encrypted WebSocket</text>
|
||||||
|
<rect x="630" width="12" height="12" rx="2" fill="#5c9cf5"/><text x="648" y="11" fill="#ccc" font-size="10">SWIM Gossip Protocol</text>
|
||||||
|
</g>
|
||||||
|
</g>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 6.2 KiB |
@@ -0,0 +1,316 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="generator" content="pandoc" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
|
||||||
|
<title>OptimClaw - lazy-tools</title>
|
||||||
|
<style>
|
||||||
|
/* Default styles provided by pandoc.
|
||||||
|
** See https://pandoc.org/MANUAL.html#variables-for-html for config info.
|
||||||
|
*/
|
||||||
|
span.smallcaps{font-variant: small-caps;}
|
||||||
|
div.columns{display: flex; gap: min(4vw, 1.5em);}
|
||||||
|
div.column{flex: auto; overflow-x: auto;}
|
||||||
|
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
|
||||||
|
/* The extra [class] is a hack that increases specificity enough to
|
||||||
|
override a similar rule in reveal.js */
|
||||||
|
ul.task-list[class]{list-style: none;}
|
||||||
|
ul.task-list li input[type="checkbox"] {
|
||||||
|
font-size: inherit;
|
||||||
|
width: 0.8em;
|
||||||
|
margin: 0 0.8em 0.2em -1.6em;
|
||||||
|
vertical-align: middle;
|
||||||
|
}
|
||||||
|
.display.math{display: block; text-align: center; margin: 0.5rem auto;}
|
||||||
|
/* CSS for syntax highlighting */
|
||||||
|
html { -webkit-text-size-adjust: 100%; }
|
||||||
|
pre > code.sourceCode { white-space: pre; position: relative; }
|
||||||
|
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
|
||||||
|
pre > code.sourceCode > span:empty { height: 1.2em; }
|
||||||
|
.sourceCode { overflow: visible; }
|
||||||
|
code.sourceCode > span { color: inherit; text-decoration: inherit; }
|
||||||
|
div.sourceCode { margin: 1em 0; }
|
||||||
|
pre.sourceCode { margin: 0; }
|
||||||
|
@media screen {
|
||||||
|
div.sourceCode { overflow: auto; }
|
||||||
|
}
|
||||||
|
@media print {
|
||||||
|
pre > code.sourceCode { white-space: pre-wrap; }
|
||||||
|
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
|
||||||
|
}
|
||||||
|
pre.numberSource code
|
||||||
|
{ counter-reset: source-line 0; }
|
||||||
|
pre.numberSource code > span
|
||||||
|
{ position: relative; left: -4em; counter-increment: source-line; }
|
||||||
|
pre.numberSource code > span > a:first-child::before
|
||||||
|
{ content: counter(source-line);
|
||||||
|
position: relative; left: -1em; text-align: right; vertical-align: baseline;
|
||||||
|
border: none; display: inline-block;
|
||||||
|
-webkit-touch-callout: none; -webkit-user-select: none;
|
||||||
|
-khtml-user-select: none; -moz-user-select: none;
|
||||||
|
-ms-user-select: none; user-select: none;
|
||||||
|
padding: 0 4px; width: 4em;
|
||||||
|
color: #aaaaaa;
|
||||||
|
}
|
||||||
|
pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; }
|
||||||
|
div.sourceCode
|
||||||
|
{ }
|
||||||
|
@media screen {
|
||||||
|
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
|
||||||
|
}
|
||||||
|
code span.al { color: #ff0000; font-weight: bold; } /* Alert */
|
||||||
|
code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */
|
||||||
|
code span.at { color: #7d9029; } /* Attribute */
|
||||||
|
code span.bn { color: #40a070; } /* BaseN */
|
||||||
|
code span.bu { color: #008000; } /* BuiltIn */
|
||||||
|
code span.cf { color: #007020; font-weight: bold; } /* ControlFlow */
|
||||||
|
code span.ch { color: #4070a0; } /* Char */
|
||||||
|
code span.cn { color: #880000; } /* Constant */
|
||||||
|
code span.co { color: #60a0b0; font-style: italic; } /* Comment */
|
||||||
|
code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */
|
||||||
|
code span.do { color: #ba2121; font-style: italic; } /* Documentation */
|
||||||
|
code span.dt { color: #902000; } /* DataType */
|
||||||
|
code span.dv { color: #40a070; } /* DecVal */
|
||||||
|
code span.er { color: #ff0000; font-weight: bold; } /* Error */
|
||||||
|
code span.ex { } /* Extension */
|
||||||
|
code span.fl { color: #40a070; } /* Float */
|
||||||
|
code span.fu { color: #06287e; } /* Function */
|
||||||
|
code span.im { color: #008000; font-weight: bold; } /* Import */
|
||||||
|
code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */
|
||||||
|
code span.kw { color: #007020; font-weight: bold; } /* Keyword */
|
||||||
|
code span.op { color: #666666; } /* Operator */
|
||||||
|
code span.ot { color: #007020; } /* Other */
|
||||||
|
code span.pp { color: #bc7a00; } /* Preprocessor */
|
||||||
|
code span.sc { color: #4070a0; } /* SpecialChar */
|
||||||
|
code span.ss { color: #bb6688; } /* SpecialString */
|
||||||
|
code span.st { color: #4070a0; } /* String */
|
||||||
|
code span.va { color: #19177c; } /* Variable */
|
||||||
|
code span.vs { color: #4070a0; } /* VerbatimString */
|
||||||
|
code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */
|
||||||
|
</style>
|
||||||
|
<link rel="stylesheet" href="data:text/css,body{font-family:Inter,system-ui,sans-serif;max-width:800px;margin:0 auto;padding:40px;color:%23e0e0e0;background:%231a1a2e}h1,h2,h3{color:%23fab283}code{background:%230f3460;padding:2px 6px;border-radius:3px;color:%2300d4aa}pre{background:%230f3460;padding:16px;border-radius:8px;overflow-x:auto}pre code{background:none}a{color:%235c9cf5}table{border-collapse:collapse;width:100%}th,td{border:1px solid%2330363d;padding:8px;text-align:left}th{background:%230f3460}" />
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<header id="title-block-header">
|
||||||
|
<h1 class="title">OptimClaw - lazy-tools</h1>
|
||||||
|
</header>
|
||||||
|
<h1 id="optimclaw-lazy-tool-loading">OptimClaw Lazy Tool Loading</h1>
|
||||||
|
<h2 id="overview">Overview</h2>
|
||||||
|
<p>Lazy tool loading is an optimization that dramatically reduces the
|
||||||
|
initial system prompt size sent to the LLM. Instead of injecting full
|
||||||
|
JSON schemas for every available tool into each request, OptimClaw loads
|
||||||
|
only a small core set of tools eagerly and defers the rest until the LLM
|
||||||
|
requests them.</p>
|
||||||
|
<p><strong>Impact:</strong> System prompt size drops from approximately
|
||||||
|
13,000 tokens to approximately 4,000 tokens – a 70% reduction. This
|
||||||
|
saves cost on every LLM call and leaves more of the context window
|
||||||
|
available for conversation history and tool outputs.</p>
|
||||||
|
<h2 id="how-to-enable">How to Enable</h2>
|
||||||
|
<p>Set the environment variable:</p>
|
||||||
|
<div class="sourceCode" id="cb1"><pre
|
||||||
|
class="sourceCode bash"><code class="sourceCode bash"><span id="cb1-1"><a href="#cb1-1" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">OPTIMCLAW_LAZY_TOOLS</span><span class="op">=</span>1</span></code></pre></div>
|
||||||
|
<p>Or add it to <code>~/.optimclaw/.env</code>:</p>
|
||||||
|
<pre class="env"><code>OPTIMCLAW_LAZY_TOOLS=1</code></pre>
|
||||||
|
<p>To disable (default behavior – all tools loaded eagerly):</p>
|
||||||
|
<div class="sourceCode" id="cb3"><pre
|
||||||
|
class="sourceCode bash"><code class="sourceCode bash"><span id="cb3-1"><a href="#cb3-1" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">OPTIMCLAW_LAZY_TOOLS</span><span class="op">=</span>0</span>
|
||||||
|
<span id="cb3-2"><a href="#cb3-2" aria-hidden="true" tabindex="-1"></a><span class="co"># or simply unset it</span></span>
|
||||||
|
<span id="cb3-3"><a href="#cb3-3" aria-hidden="true" tabindex="-1"></a><span class="bu">unset</span> <span class="va">OPTIMCLAW_LAZY_TOOLS</span></span></code></pre></div>
|
||||||
|
<h2 id="core-tools-always-loaded">Core Tools (Always Loaded)</h2>
|
||||||
|
<p>When lazy loading is enabled, the following 12 core tools are always
|
||||||
|
included in the system prompt. These are the tools the LLM needs most
|
||||||
|
frequently and cover the essential interaction patterns:</p>
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>#</th>
|
||||||
|
<th>Tool</th>
|
||||||
|
<th>Purpose</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td>1</td>
|
||||||
|
<td><code>echo</code></td>
|
||||||
|
<td>Return text to the user</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>2</td>
|
||||||
|
<td><code>time</code></td>
|
||||||
|
<td>Get current date and time</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>3</td>
|
||||||
|
<td><code>json</code></td>
|
||||||
|
<td>Parse and query JSON data</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>4</td>
|
||||||
|
<td><code>http</code></td>
|
||||||
|
<td>Make HTTP requests to allowed endpoints</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>5</td>
|
||||||
|
<td><code>web_fetch</code></td>
|
||||||
|
<td>Fetch and extract content from web pages</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>6</td>
|
||||||
|
<td><code>file_read</code></td>
|
||||||
|
<td>Read files from the workspace</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>7</td>
|
||||||
|
<td><code>file_write</code></td>
|
||||||
|
<td>Write files to the workspace</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>8</td>
|
||||||
|
<td><code>shell</code></td>
|
||||||
|
<td>Execute shell commands in the sandbox</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>9</td>
|
||||||
|
<td><code>memory_search</code></td>
|
||||||
|
<td>Search persistent memory (hybrid FTS + vector)</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>10</td>
|
||||||
|
<td><code>memory_write</code></td>
|
||||||
|
<td>Write to persistent memory</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>11</td>
|
||||||
|
<td><code>message</code></td>
|
||||||
|
<td>Send messages to channels</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>12</td>
|
||||||
|
<td><code>tool_info</code></td>
|
||||||
|
<td>Discover and load additional tools on demand</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
<h2 id="tool-discovery-with-tool_info">Tool Discovery with
|
||||||
|
tool_info</h2>
|
||||||
|
<p>The <code>tool_info</code> tool is the mechanism by which the LLM
|
||||||
|
discovers and loads deferred tools. When the LLM determines it needs a
|
||||||
|
tool that is not in its current context, it calls <code>tool_info</code>
|
||||||
|
to retrieve the full schema.</p>
|
||||||
|
<h3 id="how-it-works">How It Works</h3>
|
||||||
|
<ol type="1">
|
||||||
|
<li>The system prompt includes a brief note listing the names of all
|
||||||
|
available (but not yet loaded) tools.</li>
|
||||||
|
<li>When the LLM needs one of these tools, it calls
|
||||||
|
<code>tool_info</code> with the tool name or a search query.</li>
|
||||||
|
<li><code>tool_info</code> returns the full JSON schema (parameters,
|
||||||
|
description, examples) for the matched tools.</li>
|
||||||
|
<li>The LLM can then call the newly loaded tool in subsequent
|
||||||
|
turns.</li>
|
||||||
|
</ol>
|
||||||
|
<h3 id="tool_info-parameters">tool_info Parameters</h3>
|
||||||
|
<div class="sourceCode" id="cb4"><pre
|
||||||
|
class="sourceCode json"><code class="sourceCode json"><span id="cb4-1"><a href="#cb4-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span></span>
|
||||||
|
<span id="cb4-2"><a href="#cb4-2" aria-hidden="true" tabindex="-1"></a> <span class="dt">"name"</span><span class="fu">:</span> <span class="st">"tool_info"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb4-3"><a href="#cb4-3" aria-hidden="true" tabindex="-1"></a> <span class="dt">"parameters"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||||
|
<span id="cb4-4"><a href="#cb4-4" aria-hidden="true" tabindex="-1"></a> <span class="dt">"query"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||||
|
<span id="cb4-5"><a href="#cb4-5" aria-hidden="true" tabindex="-1"></a> <span class="dt">"type"</span><span class="fu">:</span> <span class="st">"string"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb4-6"><a href="#cb4-6" aria-hidden="true" tabindex="-1"></a> <span class="dt">"description"</span><span class="fu">:</span> <span class="st">"Exact tool name or keyword search query"</span></span>
|
||||||
|
<span id="cb4-7"><a href="#cb4-7" aria-hidden="true" tabindex="-1"></a> <span class="fu">},</span></span>
|
||||||
|
<span id="cb4-8"><a href="#cb4-8" aria-hidden="true" tabindex="-1"></a> <span class="dt">"max_results"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||||
|
<span id="cb4-9"><a href="#cb4-9" aria-hidden="true" tabindex="-1"></a> <span class="dt">"type"</span><span class="fu">:</span> <span class="st">"number"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb4-10"><a href="#cb4-10" aria-hidden="true" tabindex="-1"></a> <span class="dt">"description"</span><span class="fu">:</span> <span class="st">"Maximum tools to return (default: 5)"</span></span>
|
||||||
|
<span id="cb4-11"><a href="#cb4-11" aria-hidden="true" tabindex="-1"></a> <span class="fu">}</span></span>
|
||||||
|
<span id="cb4-12"><a href="#cb4-12" aria-hidden="true" tabindex="-1"></a> <span class="fu">}</span></span>
|
||||||
|
<span id="cb4-13"><a href="#cb4-13" aria-hidden="true" tabindex="-1"></a><span class="fu">}</span></span></code></pre></div>
|
||||||
|
<h3 id="example-flow">Example Flow</h3>
|
||||||
|
<p><strong>System prompt includes:</strong> > Additional tools
|
||||||
|
available (use <code>tool_info</code> to load): <code>job_create</code>,
|
||||||
|
<code>job_status</code>, <code>job_cancel</code>,
|
||||||
|
<code>routine_create</code>, <code>routine_list</code>,
|
||||||
|
<code>skill_search</code>, <code>skill_install</code>,
|
||||||
|
<code>extension_install</code>, <code>secrets_set</code>,
|
||||||
|
<code>secrets_get</code>, …</p>
|
||||||
|
<p><strong>LLM decides it needs to create a background job:</strong></p>
|
||||||
|
<pre><code>LLM -> tool_info(query="job_create")
|
||||||
|
|
||||||
|
tool_info returns:
|
||||||
|
{
|
||||||
|
"tools": [{
|
||||||
|
"name": "job_create",
|
||||||
|
"description": "Create a new background job with the given prompt and priority",
|
||||||
|
"parameters": {
|
||||||
|
"prompt": { "type": "string", "required": true },
|
||||||
|
"priority": { "type": "number", "default": 5 },
|
||||||
|
"timeout_secs": { "type": "number", "default": 300 }
|
||||||
|
}
|
||||||
|
}]
|
||||||
|
}
|
||||||
|
|
||||||
|
LLM -> job_create(prompt="Summarize today's news", priority=3)</code></pre>
|
||||||
|
<h2 id="when-to-use-lazy-loading">When to Use Lazy Loading</h2>
|
||||||
|
<table>
|
||||||
|
<colgroup>
|
||||||
|
<col style="width: 38%" />
|
||||||
|
<col style="width: 61%" />
|
||||||
|
</colgroup>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Scenario</th>
|
||||||
|
<th>Recommendation</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td>Production deployment with many tools/MCP servers</td>
|
||||||
|
<td>Enable – significant token savings</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Development and debugging</td>
|
||||||
|
<td>Disable – easier to see all available tools</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Cost-sensitive usage with expensive models</td>
|
||||||
|
<td>Enable – reduces per-request cost</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Clusters with heterogeneous tool sets</td>
|
||||||
|
<td>Enable – each node may have different tools</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Simple setups with few tools (<15 total)</td>
|
||||||
|
<td>Either – minimal difference</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
<h2 id="performance-characteristics">Performance Characteristics</h2>
|
||||||
|
<ul>
|
||||||
|
<li><strong>First request:</strong> Faster, because the system prompt is
|
||||||
|
smaller and the LLM processes fewer tokens.</li>
|
||||||
|
<li><strong>Tool discovery round-trip:</strong> When the LLM calls
|
||||||
|
<code>tool_info</code>, it adds one extra turn before the actual tool
|
||||||
|
call. In practice this is rare because the 12 core tools handle the
|
||||||
|
majority of interactions.</li>
|
||||||
|
<li><strong>Subsequent requests in the same session:</strong> Tool
|
||||||
|
schemas loaded via <code>tool_info</code> remain in the conversation
|
||||||
|
context for the duration of the session, so discovery cost is paid at
|
||||||
|
most once per tool per session.</li>
|
||||||
|
</ul>
|
||||||
|
<h2 id="interaction-with-other-features">Interaction with Other
|
||||||
|
Features</h2>
|
||||||
|
<ul>
|
||||||
|
<li><strong>MCP tools:</strong> MCP-connected tool schemas are also
|
||||||
|
deferred when lazy loading is enabled. They appear in the “additional
|
||||||
|
tools available” list and are loaded via <code>tool_info</code>.</li>
|
||||||
|
<li><strong>WASM tools:</strong> Same behavior as MCP tools – deferred
|
||||||
|
and discoverable.</li>
|
||||||
|
<li><strong>Skills:</strong> Skill tools (<code>skill_list</code>,
|
||||||
|
<code>skill_search</code>, <code>skill_install</code>,
|
||||||
|
<code>skill_remove</code>) are deferred. The skill system itself is
|
||||||
|
unaffected.</li>
|
||||||
|
<li><strong>Mesh cluster:</strong> Lazy loading is a per-node setting.
|
||||||
|
Different nodes in a cluster can have different settings.</li>
|
||||||
|
</ul>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -0,0 +1,786 @@
|
|||||||
|
<!DOCTYPE html>
|
||||||
|
<html xmlns="http://www.w3.org/1999/xhtml">
|
||||||
|
<head>
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta name="generator" content="pandoc" />
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
|
||||||
|
<title>OptimClaw - mesh-cluster</title>
|
||||||
|
<style>
|
||||||
|
/* Default styles provided by pandoc.
|
||||||
|
** See https://pandoc.org/MANUAL.html#variables-for-html for config info.
|
||||||
|
*/
|
||||||
|
span.smallcaps{font-variant: small-caps;}
|
||||||
|
div.columns{display: flex; gap: min(4vw, 1.5em);}
|
||||||
|
div.column{flex: auto; overflow-x: auto;}
|
||||||
|
div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;}
|
||||||
|
/* The extra [class] is a hack that increases specificity enough to
|
||||||
|
override a similar rule in reveal.js */
|
||||||
|
ul.task-list[class]{list-style: none;}
|
||||||
|
ul.task-list li input[type="checkbox"] {
|
||||||
|
font-size: inherit;
|
||||||
|
width: 0.8em;
|
||||||
|
margin: 0 0.8em 0.2em -1.6em;
|
||||||
|
vertical-align: middle;
|
||||||
|
}
|
||||||
|
.display.math{display: block; text-align: center; margin: 0.5rem auto;}
|
||||||
|
/* CSS for syntax highlighting */
|
||||||
|
html { -webkit-text-size-adjust: 100%; }
|
||||||
|
pre > code.sourceCode { white-space: pre; position: relative; }
|
||||||
|
pre > code.sourceCode > span { display: inline-block; line-height: 1.25; }
|
||||||
|
pre > code.sourceCode > span:empty { height: 1.2em; }
|
||||||
|
.sourceCode { overflow: visible; }
|
||||||
|
code.sourceCode > span { color: inherit; text-decoration: inherit; }
|
||||||
|
div.sourceCode { margin: 1em 0; }
|
||||||
|
pre.sourceCode { margin: 0; }
|
||||||
|
@media screen {
|
||||||
|
div.sourceCode { overflow: auto; }
|
||||||
|
}
|
||||||
|
@media print {
|
||||||
|
pre > code.sourceCode { white-space: pre-wrap; }
|
||||||
|
pre > code.sourceCode > span { text-indent: -5em; padding-left: 5em; }
|
||||||
|
}
|
||||||
|
pre.numberSource code
|
||||||
|
{ counter-reset: source-line 0; }
|
||||||
|
pre.numberSource code > span
|
||||||
|
{ position: relative; left: -4em; counter-increment: source-line; }
|
||||||
|
pre.numberSource code > span > a:first-child::before
|
||||||
|
{ content: counter(source-line);
|
||||||
|
position: relative; left: -1em; text-align: right; vertical-align: baseline;
|
||||||
|
border: none; display: inline-block;
|
||||||
|
-webkit-touch-callout: none; -webkit-user-select: none;
|
||||||
|
-khtml-user-select: none; -moz-user-select: none;
|
||||||
|
-ms-user-select: none; user-select: none;
|
||||||
|
padding: 0 4px; width: 4em;
|
||||||
|
color: #aaaaaa;
|
||||||
|
}
|
||||||
|
pre.numberSource { margin-left: 3em; border-left: 1px solid #aaaaaa; padding-left: 4px; }
|
||||||
|
div.sourceCode
|
||||||
|
{ }
|
||||||
|
@media screen {
|
||||||
|
pre > code.sourceCode > span > a:first-child::before { text-decoration: underline; }
|
||||||
|
}
|
||||||
|
code span.al { color: #ff0000; font-weight: bold; } /* Alert */
|
||||||
|
code span.an { color: #60a0b0; font-weight: bold; font-style: italic; } /* Annotation */
|
||||||
|
code span.at { color: #7d9029; } /* Attribute */
|
||||||
|
code span.bn { color: #40a070; } /* BaseN */
|
||||||
|
code span.bu { color: #008000; } /* BuiltIn */
|
||||||
|
code span.cf { color: #007020; font-weight: bold; } /* ControlFlow */
|
||||||
|
code span.ch { color: #4070a0; } /* Char */
|
||||||
|
code span.cn { color: #880000; } /* Constant */
|
||||||
|
code span.co { color: #60a0b0; font-style: italic; } /* Comment */
|
||||||
|
code span.cv { color: #60a0b0; font-weight: bold; font-style: italic; } /* CommentVar */
|
||||||
|
code span.do { color: #ba2121; font-style: italic; } /* Documentation */
|
||||||
|
code span.dt { color: #902000; } /* DataType */
|
||||||
|
code span.dv { color: #40a070; } /* DecVal */
|
||||||
|
code span.er { color: #ff0000; font-weight: bold; } /* Error */
|
||||||
|
code span.ex { } /* Extension */
|
||||||
|
code span.fl { color: #40a070; } /* Float */
|
||||||
|
code span.fu { color: #06287e; } /* Function */
|
||||||
|
code span.im { color: #008000; font-weight: bold; } /* Import */
|
||||||
|
code span.in { color: #60a0b0; font-weight: bold; font-style: italic; } /* Information */
|
||||||
|
code span.kw { color: #007020; font-weight: bold; } /* Keyword */
|
||||||
|
code span.op { color: #666666; } /* Operator */
|
||||||
|
code span.ot { color: #007020; } /* Other */
|
||||||
|
code span.pp { color: #bc7a00; } /* Preprocessor */
|
||||||
|
code span.sc { color: #4070a0; } /* SpecialChar */
|
||||||
|
code span.ss { color: #bb6688; } /* SpecialString */
|
||||||
|
code span.st { color: #4070a0; } /* String */
|
||||||
|
code span.va { color: #19177c; } /* Variable */
|
||||||
|
code span.vs { color: #4070a0; } /* VerbatimString */
|
||||||
|
code span.wa { color: #60a0b0; font-weight: bold; font-style: italic; } /* Warning */
|
||||||
|
</style>
|
||||||
|
<link rel="stylesheet" href="data:text/css,body{font-family:Inter,system-ui,sans-serif;max-width:800px;margin:0 auto;padding:40px;color:%23e0e0e0;background:%231a1a2e}h1,h2,h3{color:%23fab283}code{background:%230f3460;padding:2px 6px;border-radius:3px;color:%2300d4aa}pre{background:%230f3460;padding:16px;border-radius:8px;overflow-x:auto}pre code{background:none}a{color:%235c9cf5}table{border-collapse:collapse;width:100%}th,td{border:1px solid%2330363d;padding:8px;text-align:left}th{background:%230f3460}" />
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<header id="title-block-header">
|
||||||
|
<h1 class="title">OptimClaw - mesh-cluster</h1>
|
||||||
|
</header>
|
||||||
|
<h1 id="optimclaw-mesh-cluster">OptimClaw Mesh Cluster</h1>
|
||||||
|
<h2 id="overview">Overview</h2>
|
||||||
|
<p>The OptimClaw Mesh Cluster enables multiple OptimClaw instances to
|
||||||
|
form an autonomous AI mesh network. Each node in the cluster operates
|
||||||
|
independently while collaborating on tasks, sharing workload, and
|
||||||
|
providing fault tolerance. Nodes discover each other automatically via
|
||||||
|
UDP beacons, authenticate using post-quantum cryptography, and
|
||||||
|
coordinate through a gossip-based membership protocol.</p>
|
||||||
|
<p>Key capabilities:</p>
|
||||||
|
<ul>
|
||||||
|
<li><strong>Automatic discovery</strong> – zero-configuration node
|
||||||
|
joining via UDP broadcast beacons</li>
|
||||||
|
<li><strong>Post-quantum security</strong> – ML-KEM-768 key
|
||||||
|
encapsulation with AES-256-GCM authenticated encryption</li>
|
||||||
|
<li><strong>Gossip membership</strong> – SWIM protocol for reliable
|
||||||
|
failure detection and cluster state convergence</li>
|
||||||
|
<li><strong>Intelligent task routing</strong> – scoring algorithm that
|
||||||
|
considers load, latency, capability, and affinity</li>
|
||||||
|
<li><strong>Graceful degradation</strong> – nodes operate independently
|
||||||
|
if connectivity is lost</li>
|
||||||
|
</ul>
|
||||||
|
<h2 id="architecture">Architecture</h2>
|
||||||
|
<pre><code> ┌─────────────────────────────────────────────┐
|
||||||
|
│ Mesh Cluster │
|
||||||
|
│ │
|
||||||
|
┌──────────────┐ │ ┌──────────┐ Gossip ┌──────────┐ │
|
||||||
|
│ Client │──────►│ │ Node A │◄────────────►│ Node B │ │
|
||||||
|
│ (any chan.) │ │ │ │ (SWIM) │ │ │
|
||||||
|
└──────────────┘ │ │ ┌──────┐ │ │ ┌──────┐ │ │
|
||||||
|
│ │ │Agent │ │ │ │Agent │ │ │
|
||||||
|
│ │ │ Loop │ │ │ │ Loop │ │ │
|
||||||
|
│ │ └──────┘ │ │ └──────┘ │ │
|
||||||
|
│ │ ┌──────┐ │ │ ┌──────┐ │ │
|
||||||
|
│ │ │Tools │ │ │ │Tools │ │ │
|
||||||
|
│ │ └──────┘ │ │ └──────┘ │ │
|
||||||
|
│ └─────┬────┘ └────┬─────┘ │
|
||||||
|
│ │ │ │
|
||||||
|
│ │ UDP Beacons │ │
|
||||||
|
│ │◄───────────────────────►│ │
|
||||||
|
│ │ │ │
|
||||||
|
│ │ Task Routing │ │
|
||||||
|
│ │◄───────────────────────►│ │
|
||||||
|
│ │ (ML-KEM-768 + │ │
|
||||||
|
│ │ AES-256-GCM) │ │
|
||||||
|
│ ┌─────┴────┐ ┌────┴─────┐ │
|
||||||
|
│ │ Node C │◄────────────►│ Node D │ │
|
||||||
|
│ └──────────┘ Gossip └──────────┘ │
|
||||||
|
│ │
|
||||||
|
└─────────────────────────────────────────────┘
|
||||||
|
|
||||||
|
Data flow:
|
||||||
|
1. UDP beacon broadcast → node discovery
|
||||||
|
2. ML-KEM-768 handshake → shared secret
|
||||||
|
3. AES-256-GCM encrypted channel established
|
||||||
|
4. SWIM gossip protocol → membership state
|
||||||
|
5. Task routing → best node selected via scoring
|
||||||
|
6. Encrypted task dispatch + result collection</code></pre>
|
||||||
|
<h2 id="configuration">Configuration</h2>
|
||||||
|
<p>All cluster settings are controlled via environment variables
|
||||||
|
prefixed with <code>CLUSTER_</code>. They can be set in
|
||||||
|
<code>~/.optimclaw/.env</code> or passed directly.</p>
|
||||||
|
<table>
|
||||||
|
<colgroup>
|
||||||
|
<col style="width: 26%" />
|
||||||
|
<col style="width: 15%" />
|
||||||
|
<col style="width: 23%" />
|
||||||
|
<col style="width: 34%" />
|
||||||
|
</colgroup>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Variable</th>
|
||||||
|
<th>Type</th>
|
||||||
|
<th>Default</th>
|
||||||
|
<th>Description</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_ENABLED</code></td>
|
||||||
|
<td>bool</td>
|
||||||
|
<td><code>false</code></td>
|
||||||
|
<td>Enable mesh cluster mode</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_NODE_ID</code></td>
|
||||||
|
<td>string</td>
|
||||||
|
<td>auto (hostname)</td>
|
||||||
|
<td>Unique identifier for this node</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_BIND_ADDR</code></td>
|
||||||
|
<td>string</td>
|
||||||
|
<td><code>0.0.0.0</code></td>
|
||||||
|
<td>Address to bind the cluster transport</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_BIND_PORT</code></td>
|
||||||
|
<td>u16</td>
|
||||||
|
<td><code>9400</code></td>
|
||||||
|
<td>Port for the encrypted cluster transport</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_BEACON_PORT</code></td>
|
||||||
|
<td>u16</td>
|
||||||
|
<td><code>9401</code></td>
|
||||||
|
<td>UDP port for discovery beacons</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_BEACON_INTERVAL_MS</code></td>
|
||||||
|
<td>u64</td>
|
||||||
|
<td><code>5000</code></td>
|
||||||
|
<td>Milliseconds between beacon broadcasts</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_BEACON_SUBNET</code></td>
|
||||||
|
<td>string</td>
|
||||||
|
<td><code>255.255.255.255</code></td>
|
||||||
|
<td>Broadcast address for beacons</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_SECRET</code></td>
|
||||||
|
<td>string</td>
|
||||||
|
<td><em>required</em></td>
|
||||||
|
<td>Pre-shared key for cluster authentication (min 32 chars)</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_SEEDS</code></td>
|
||||||
|
<td>string</td>
|
||||||
|
<td><code>""</code></td>
|
||||||
|
<td>Comma-separated seed node addresses (<code>host:port</code>) for
|
||||||
|
non-broadcast environments</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_GOSSIP_INTERVAL_MS</code></td>
|
||||||
|
<td>u64</td>
|
||||||
|
<td><code>1000</code></td>
|
||||||
|
<td>Milliseconds between gossip protocol rounds</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_GOSSIP_FANOUT</code></td>
|
||||||
|
<td>u8</td>
|
||||||
|
<td><code>3</code></td>
|
||||||
|
<td>Number of peers to gossip with per round</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_SUSPICION_MULT</code></td>
|
||||||
|
<td>u8</td>
|
||||||
|
<td><code>4</code></td>
|
||||||
|
<td>Multiplier for suspicion timeout (suspicion_mult *
|
||||||
|
gossip_interval)</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_PROBE_INTERVAL_MS</code></td>
|
||||||
|
<td>u64</td>
|
||||||
|
<td><code>2000</code></td>
|
||||||
|
<td>Milliseconds between SWIM probe pings</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_PROBE_TIMEOUT_MS</code></td>
|
||||||
|
<td>u64</td>
|
||||||
|
<td><code>500</code></td>
|
||||||
|
<td>Timeout for a direct probe response</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_INDIRECT_PROBES</code></td>
|
||||||
|
<td>u8</td>
|
||||||
|
<td><code>3</code></td>
|
||||||
|
<td>Number of indirect probes before suspicion</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_TASK_TIMEOUT_SECS</code></td>
|
||||||
|
<td>u64</td>
|
||||||
|
<td><code>300</code></td>
|
||||||
|
<td>Timeout for a routed task to complete</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_MAX_NODES</code></td>
|
||||||
|
<td>u16</td>
|
||||||
|
<td><code>64</code></td>
|
||||||
|
<td>Maximum cluster size</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_TLS_CERT</code></td>
|
||||||
|
<td>path</td>
|
||||||
|
<td><code>""</code></td>
|
||||||
|
<td>Optional TLS certificate for cross-datacenter transport</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_TLS_KEY</code></td>
|
||||||
|
<td>path</td>
|
||||||
|
<td><code>""</code></td>
|
||||||
|
<td>Optional TLS private key</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_ADVERTISE_ADDR</code></td>
|
||||||
|
<td>string</td>
|
||||||
|
<td>auto</td>
|
||||||
|
<td>Address advertised to other nodes (for NAT traversal)</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_ADVERTISE_PORT</code></td>
|
||||||
|
<td>u16</td>
|
||||||
|
<td>same as bind</td>
|
||||||
|
<td>Port advertised to other nodes</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_REGION</code></td>
|
||||||
|
<td>string</td>
|
||||||
|
<td><code>""</code></td>
|
||||||
|
<td>Logical region tag for locality-aware routing</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>CLUSTER_CAPABILITIES</code></td>
|
||||||
|
<td>string</td>
|
||||||
|
<td><code>""</code></td>
|
||||||
|
<td>Comma-separated capability tags (e.g.,
|
||||||
|
<code>gpu,high-memory,docker</code>)</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
<h3 id="minimal-configuration">Minimal Configuration</h3>
|
||||||
|
<pre class="env"><code>CLUSTER_ENABLED=true
|
||||||
|
CLUSTER_SECRET=my-very-long-pre-shared-key-at-least-32-chars</code></pre>
|
||||||
|
<h3 id="cross-datacenter-configuration">Cross-Datacenter
|
||||||
|
Configuration</h3>
|
||||||
|
<pre class="env"><code>CLUSTER_ENABLED=true
|
||||||
|
CLUSTER_SECRET=my-very-long-pre-shared-key-at-least-32-chars
|
||||||
|
CLUSTER_SEEDS=dc1-node1.example.com:9400,dc2-node1.example.com:9400
|
||||||
|
CLUSTER_ADVERTISE_ADDR=203.0.113.10
|
||||||
|
CLUSTER_REGION=us-east-1
|
||||||
|
CLUSTER_TLS_CERT=/etc/optimclaw/cluster.crt
|
||||||
|
CLUSTER_TLS_KEY=/etc/optimclaw/cluster.key</code></pre>
|
||||||
|
<h2 id="discovery-protocol">Discovery Protocol</h2>
|
||||||
|
<p>Nodes discover each other using a UDP beacon protocol. When a node
|
||||||
|
starts with <code>CLUSTER_ENABLED=true</code>, it begins broadcasting
|
||||||
|
beacon packets on the configured broadcast address and port.</p>
|
||||||
|
<h3 id="beacon-packet-format">Beacon Packet Format</h3>
|
||||||
|
<pre><code>Offset Size Field
|
||||||
|
0 4 Magic bytes: 0x4F 0x43 0x4D 0x53 ("OCMS")
|
||||||
|
4 1 Protocol version (currently 0x01)
|
||||||
|
5 2 Beacon port (big-endian u16)
|
||||||
|
7 2 Transport port (big-endian u16)
|
||||||
|
9 32 Node ID (UTF-8, zero-padded)
|
||||||
|
41 32 HMAC-SHA256 of bytes 0..41 using CLUSTER_SECRET</code></pre>
|
||||||
|
<p>Total beacon size: 73 bytes.</p>
|
||||||
|
<h3 id="discovery-sequence">Discovery Sequence</h3>
|
||||||
|
<ol type="1">
|
||||||
|
<li>On startup, the node broadcasts a beacon every
|
||||||
|
<code>CLUSTER_BEACON_INTERVAL_MS</code> milliseconds to
|
||||||
|
<code>CLUSTER_BEACON_SUBNET:CLUSTER_BEACON_PORT</code>.</li>
|
||||||
|
<li>All listening nodes receive the beacon, verify the HMAC against
|
||||||
|
their own <code>CLUSTER_SECRET</code>, and extract the sender’s
|
||||||
|
transport address.</li>
|
||||||
|
<li>If the beacon is from an unknown node, the receiving node initiates
|
||||||
|
a post-quantum key exchange (see below) over TCP to the sender’s
|
||||||
|
transport address.</li>
|
||||||
|
<li>Once the encrypted channel is established, the new node is added to
|
||||||
|
the membership list and the gossip protocol takes over.</li>
|
||||||
|
<li>In non-broadcast environments (cloud, cross-datacenter), set
|
||||||
|
<code>CLUSTER_SEEDS</code> to bootstrap. The node will contact seed
|
||||||
|
addresses directly instead of relying on broadcast.</li>
|
||||||
|
</ol>
|
||||||
|
<p>Beacons continue to be sent after joining to help new nodes discover
|
||||||
|
the cluster.</p>
|
||||||
|
<h2 id="post-quantum-cryptography">Post-Quantum Cryptography</h2>
|
||||||
|
<p>All inter-node communication is encrypted using a hybrid post-quantum
|
||||||
|
scheme to protect against both classical and quantum adversaries.</p>
|
||||||
|
<h3 id="key-exchange-ml-kem-768">Key Exchange: ML-KEM-768</h3>
|
||||||
|
<p>ML-KEM-768 (formerly CRYSTALS-Kyber) is a lattice-based key
|
||||||
|
encapsulation mechanism standardized in FIPS 203. It provides IND-CCA2
|
||||||
|
security at NIST security level 3 (roughly equivalent to AES-192).</p>
|
||||||
|
<p>The handshake proceeds as follows:</p>
|
||||||
|
<ol type="1">
|
||||||
|
<li><strong>Initiator</strong> generates an ML-KEM-768 keypair
|
||||||
|
(ephemeral) and sends the public key (1184 bytes) along with its node ID
|
||||||
|
and a challenge derived from <code>CLUSTER_SECRET</code>.</li>
|
||||||
|
<li><strong>Responder</strong> verifies the challenge, encapsulates a
|
||||||
|
shared secret using the received public key, and sends back the
|
||||||
|
ciphertext (1088 bytes) along with its own challenge response.</li>
|
||||||
|
<li>Both sides derive the same 256-bit shared secret from the ML-KEM
|
||||||
|
decapsulation.</li>
|
||||||
|
<li>The shared secret is combined with <code>CLUSTER_SECRET</code> via
|
||||||
|
HKDF-SHA256 to produce the final session key, binding the session to the
|
||||||
|
cluster identity.</li>
|
||||||
|
</ol>
|
||||||
|
<h3 id="authenticated-encryption-aes-256-gcm">Authenticated Encryption:
|
||||||
|
AES-256-GCM</h3>
|
||||||
|
<p>All messages after the handshake are encrypted with AES-256-GCM using
|
||||||
|
the derived session key:</p>
|
||||||
|
<ul>
|
||||||
|
<li><strong>Nonce</strong>: 96-bit, incremented per message (with
|
||||||
|
sender-direction bit to avoid reuse)</li>
|
||||||
|
<li><strong>AAD (Additional Authenticated Data)</strong>: message type +
|
||||||
|
sequence number + sender node ID</li>
|
||||||
|
<li><strong>Tag</strong>: 128-bit authentication tag appended to
|
||||||
|
ciphertext</li>
|
||||||
|
</ul>
|
||||||
|
<h3 id="key-rotation">Key Rotation</h3>
|
||||||
|
<p>Session keys are rotated every 1 hour or after 2^32 messages,
|
||||||
|
whichever comes first. Rotation uses a new ML-KEM-768 encapsulation
|
||||||
|
within the existing encrypted channel.</p>
|
||||||
|
<h3 id="why-post-quantum">Why Post-Quantum?</h3>
|
||||||
|
<p>Mesh clusters may carry sensitive task data (credentials, personal
|
||||||
|
information, tool outputs). Harvest-now-decrypt-later attacks make it
|
||||||
|
prudent to deploy post-quantum cryptography today, even before
|
||||||
|
large-scale quantum computers exist.</p>
|
||||||
|
<h2 id="gossip-protocol-swim-membership">Gossip Protocol (SWIM
|
||||||
|
Membership)</h2>
|
||||||
|
<p>The cluster uses the SWIM (Scalable Weakly-consistent Infection-style
|
||||||
|
process group Membership) protocol for membership management and failure
|
||||||
|
detection.</p>
|
||||||
|
<h3 id="membership-states">Membership States</h3>
|
||||||
|
<p>Each node maintains a membership list where every entry is in one of
|
||||||
|
three states:</p>
|
||||||
|
<table>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>State</th>
|
||||||
|
<th>Meaning</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td><strong>Alive</strong></td>
|
||||||
|
<td>Node is healthy and responsive</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><strong>Suspect</strong></td>
|
||||||
|
<td>Node failed to respond to probes; may be down</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><strong>Dead</strong></td>
|
||||||
|
<td>Node confirmed unreachable; removed from routing</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
<h3 id="protocol-rounds">Protocol Rounds</h3>
|
||||||
|
<p>Every <code>CLUSTER_GOSSIP_INTERVAL_MS</code>, each node
|
||||||
|
performs:</p>
|
||||||
|
<ol type="1">
|
||||||
|
<li><strong>Probe</strong> – Select a random alive member and send a
|
||||||
|
direct ping. If no ack within <code>CLUSTER_PROBE_TIMEOUT_MS</code>,
|
||||||
|
send indirect pings through <code>CLUSTER_INDIRECT_PROBES</code> random
|
||||||
|
members. If still no ack, mark the target as Suspect.</li>
|
||||||
|
<li><strong>Gossip</strong> – Piggyback membership updates (state
|
||||||
|
changes, join/leave events) on probe messages. Each update includes a
|
||||||
|
Lamport timestamp for crdt-style conflict resolution.</li>
|
||||||
|
<li><strong>Suspicion</strong> – Suspect nodes have
|
||||||
|
<code>CLUSTER_SUSPICION_MULT * CLUSTER_GOSSIP_INTERVAL_MS</code> to
|
||||||
|
refute by sending an Alive message with a higher incarnation number. If
|
||||||
|
not refuted, the node transitions to Dead.</li>
|
||||||
|
</ol>
|
||||||
|
<h3 id="consistency">Consistency</h3>
|
||||||
|
<p>SWIM provides eventual consistency. After a state change, all nodes
|
||||||
|
converge within O(log N) gossip rounds, where N is the cluster size.
|
||||||
|
With default settings (1s gossip interval, fanout 3), a 64-node cluster
|
||||||
|
converges in under 7 seconds.</p>
|
||||||
|
<h3 id="join-and-leave">Join and Leave</h3>
|
||||||
|
<ul>
|
||||||
|
<li><strong>Join</strong>: Triggered by beacon discovery or seed
|
||||||
|
contact. The joining node sends a Join message; existing members
|
||||||
|
propagate the new membership via gossip.</li>
|
||||||
|
<li><strong>Graceful leave</strong>: A node sends a Leave message before
|
||||||
|
shutting down. Other nodes immediately mark it Dead without
|
||||||
|
suspicion.</li>
|
||||||
|
<li><strong>Crash</strong>: Detected by the probe/suspicion mechanism
|
||||||
|
described above.</li>
|
||||||
|
</ul>
|
||||||
|
<h2 id="task-routing-algorithm">Task Routing Algorithm</h2>
|
||||||
|
<p>When a task arrives at any node, the router decides whether to
|
||||||
|
execute it locally or forward it to a better-suited node. The decision
|
||||||
|
is based on a scoring formula applied to each alive node.</p>
|
||||||
|
<h3 id="scoring-formula">Scoring Formula</h3>
|
||||||
|
<pre><code>score(node) = w_load * (1 - load_ratio)
|
||||||
|
+ w_latency * (1 - latency_ratio)
|
||||||
|
+ w_capability * capability_match
|
||||||
|
+ w_affinity * affinity_bonus
|
||||||
|
+ w_locality * locality_bonus</code></pre>
|
||||||
|
<p>Where:</p>
|
||||||
|
<table>
|
||||||
|
<colgroup>
|
||||||
|
<col style="width: 20%" />
|
||||||
|
<col style="width: 47%" />
|
||||||
|
<col style="width: 32%" />
|
||||||
|
</colgroup>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Factor</th>
|
||||||
|
<th>Weight (default)</th>
|
||||||
|
<th>Description</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td><code>load_ratio</code></td>
|
||||||
|
<td><code>w_load = 0.35</code></td>
|
||||||
|
<td>Current jobs / max parallel jobs (lower is better)</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>latency_ratio</code></td>
|
||||||
|
<td><code>w_latency = 0.25</code></td>
|
||||||
|
<td>P95 RTT to this node / max observed RTT (lower is better)</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>capability_match</code></td>
|
||||||
|
<td><code>w_capability = 0.25</code></td>
|
||||||
|
<td>1.0 if node has all required capabilities, 0.0 otherwise</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>affinity_bonus</code></td>
|
||||||
|
<td><code>w_affinity = 0.10</code></td>
|
||||||
|
<td>1.0 if the task has session affinity to this node, 0.0
|
||||||
|
otherwise</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td><code>locality_bonus</code></td>
|
||||||
|
<td><code>w_locality = 0.05</code></td>
|
||||||
|
<td>1.0 if same <code>CLUSTER_REGION</code>, 0.5 if no region set, 0.0
|
||||||
|
otherwise</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
<h3 id="routing-decision">Routing Decision</h3>
|
||||||
|
<ol type="1">
|
||||||
|
<li>Compute <code>score(node)</code> for all alive nodes including
|
||||||
|
self.</li>
|
||||||
|
<li>If the local node’s score is within 10% of the best score, execute
|
||||||
|
locally (avoids unnecessary forwarding overhead).</li>
|
||||||
|
<li>Otherwise, forward the task to the highest-scoring node over the
|
||||||
|
encrypted channel.</li>
|
||||||
|
<li>If the target node fails to accept within 5 seconds, fall back to
|
||||||
|
local execution.</li>
|
||||||
|
<li>Results are returned to the originating node and delivered to the
|
||||||
|
original client.</li>
|
||||||
|
</ol>
|
||||||
|
<h3 id="session-affinity">Session Affinity</h3>
|
||||||
|
<p>Tasks that reference an ongoing conversation or job context are
|
||||||
|
preferentially routed to the node that holds that context. This avoids
|
||||||
|
expensive context transfer between nodes.</p>
|
||||||
|
<h2 id="api-endpoints">API Endpoints</h2>
|
||||||
|
<p>The mesh cluster exposes monitoring endpoints on the standard web
|
||||||
|
gateway.</p>
|
||||||
|
<h3 id="get-apimeshstatus">GET /api/mesh/status</h3>
|
||||||
|
<p>Returns the cluster status for the local node.</p>
|
||||||
|
<p><strong>Response:</strong></p>
|
||||||
|
<div class="sourceCode" id="cb6"><pre
|
||||||
|
class="sourceCode json"><code class="sourceCode json"><span id="cb6-1"><a href="#cb6-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span></span>
|
||||||
|
<span id="cb6-2"><a href="#cb6-2" aria-hidden="true" tabindex="-1"></a> <span class="dt">"cluster_enabled"</span><span class="fu">:</span> <span class="kw">true</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-3"><a href="#cb6-3" aria-hidden="true" tabindex="-1"></a> <span class="dt">"node_id"</span><span class="fu">:</span> <span class="st">"node-alpha"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-4"><a href="#cb6-4" aria-hidden="true" tabindex="-1"></a> <span class="dt">"state"</span><span class="fu">:</span> <span class="st">"alive"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-5"><a href="#cb6-5" aria-hidden="true" tabindex="-1"></a> <span class="dt">"region"</span><span class="fu">:</span> <span class="st">"us-east-1"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-6"><a href="#cb6-6" aria-hidden="true" tabindex="-1"></a> <span class="dt">"capabilities"</span><span class="fu">:</span> <span class="ot">[</span><span class="st">"gpu"</span><span class="ot">,</span> <span class="st">"docker"</span><span class="ot">]</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-7"><a href="#cb6-7" aria-hidden="true" tabindex="-1"></a> <span class="dt">"uptime_secs"</span><span class="fu">:</span> <span class="dv">86423</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-8"><a href="#cb6-8" aria-hidden="true" tabindex="-1"></a> <span class="dt">"transport"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||||
|
<span id="cb6-9"><a href="#cb6-9" aria-hidden="true" tabindex="-1"></a> <span class="dt">"bind_addr"</span><span class="fu">:</span> <span class="st">"0.0.0.0:9400"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-10"><a href="#cb6-10" aria-hidden="true" tabindex="-1"></a> <span class="dt">"advertise_addr"</span><span class="fu">:</span> <span class="st">"203.0.113.10:9400"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-11"><a href="#cb6-11" aria-hidden="true" tabindex="-1"></a> <span class="dt">"encryption"</span><span class="fu">:</span> <span class="st">"ML-KEM-768 + AES-256-GCM"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-12"><a href="#cb6-12" aria-hidden="true" tabindex="-1"></a> <span class="dt">"protocol_version"</span><span class="fu">:</span> <span class="dv">1</span></span>
|
||||||
|
<span id="cb6-13"><a href="#cb6-13" aria-hidden="true" tabindex="-1"></a> <span class="fu">},</span></span>
|
||||||
|
<span id="cb6-14"><a href="#cb6-14" aria-hidden="true" tabindex="-1"></a> <span class="dt">"membership"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||||
|
<span id="cb6-15"><a href="#cb6-15" aria-hidden="true" tabindex="-1"></a> <span class="dt">"alive"</span><span class="fu">:</span> <span class="dv">4</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-16"><a href="#cb6-16" aria-hidden="true" tabindex="-1"></a> <span class="dt">"suspect"</span><span class="fu">:</span> <span class="dv">0</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-17"><a href="#cb6-17" aria-hidden="true" tabindex="-1"></a> <span class="dt">"dead"</span><span class="fu">:</span> <span class="dv">1</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-18"><a href="#cb6-18" aria-hidden="true" tabindex="-1"></a> <span class="dt">"total_seen"</span><span class="fu">:</span> <span class="dv">5</span></span>
|
||||||
|
<span id="cb6-19"><a href="#cb6-19" aria-hidden="true" tabindex="-1"></a> <span class="fu">},</span></span>
|
||||||
|
<span id="cb6-20"><a href="#cb6-20" aria-hidden="true" tabindex="-1"></a> <span class="dt">"routing"</span><span class="fu">:</span> <span class="fu">{</span></span>
|
||||||
|
<span id="cb6-21"><a href="#cb6-21" aria-hidden="true" tabindex="-1"></a> <span class="dt">"local_load"</span><span class="fu">:</span> <span class="fl">0.35</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-22"><a href="#cb6-22" aria-hidden="true" tabindex="-1"></a> <span class="dt">"tasks_routed_out"</span><span class="fu">:</span> <span class="dv">142</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-23"><a href="#cb6-23" aria-hidden="true" tabindex="-1"></a> <span class="dt">"tasks_routed_in"</span><span class="fu">:</span> <span class="dv">87</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb6-24"><a href="#cb6-24" aria-hidden="true" tabindex="-1"></a> <span class="dt">"tasks_failed_over"</span><span class="fu">:</span> <span class="dv">3</span></span>
|
||||||
|
<span id="cb6-25"><a href="#cb6-25" aria-hidden="true" tabindex="-1"></a> <span class="fu">}</span></span>
|
||||||
|
<span id="cb6-26"><a href="#cb6-26" aria-hidden="true" tabindex="-1"></a><span class="fu">}</span></span></code></pre></div>
|
||||||
|
<h3 id="get-apimeshnodes">GET /api/mesh/nodes</h3>
|
||||||
|
<p>Returns the membership list with per-node details.</p>
|
||||||
|
<p><strong>Response:</strong></p>
|
||||||
|
<div class="sourceCode" id="cb7"><pre
|
||||||
|
class="sourceCode json"><code class="sourceCode json"><span id="cb7-1"><a href="#cb7-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span></span>
|
||||||
|
<span id="cb7-2"><a href="#cb7-2" aria-hidden="true" tabindex="-1"></a> <span class="dt">"nodes"</span><span class="fu">:</span> <span class="ot">[</span></span>
|
||||||
|
<span id="cb7-3"><a href="#cb7-3" aria-hidden="true" tabindex="-1"></a> <span class="fu">{</span></span>
|
||||||
|
<span id="cb7-4"><a href="#cb7-4" aria-hidden="true" tabindex="-1"></a> <span class="dt">"node_id"</span><span class="fu">:</span> <span class="st">"node-alpha"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-5"><a href="#cb7-5" aria-hidden="true" tabindex="-1"></a> <span class="dt">"state"</span><span class="fu">:</span> <span class="st">"alive"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-6"><a href="#cb7-6" aria-hidden="true" tabindex="-1"></a> <span class="dt">"addr"</span><span class="fu">:</span> <span class="st">"203.0.113.10:9400"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-7"><a href="#cb7-7" aria-hidden="true" tabindex="-1"></a> <span class="dt">"region"</span><span class="fu">:</span> <span class="st">"us-east-1"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-8"><a href="#cb7-8" aria-hidden="true" tabindex="-1"></a> <span class="dt">"capabilities"</span><span class="fu">:</span> <span class="ot">[</span><span class="st">"gpu"</span><span class="ot">,</span> <span class="st">"docker"</span><span class="ot">]</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-9"><a href="#cb7-9" aria-hidden="true" tabindex="-1"></a> <span class="dt">"load_ratio"</span><span class="fu">:</span> <span class="fl">0.35</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-10"><a href="#cb7-10" aria-hidden="true" tabindex="-1"></a> <span class="dt">"latency_ms"</span><span class="fu">:</span> <span class="dv">0</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-11"><a href="#cb7-11" aria-hidden="true" tabindex="-1"></a> <span class="dt">"last_seen"</span><span class="fu">:</span> <span class="st">"2026-03-29T12:34:56Z"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-12"><a href="#cb7-12" aria-hidden="true" tabindex="-1"></a> <span class="dt">"incarnation"</span><span class="fu">:</span> <span class="dv">7</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-13"><a href="#cb7-13" aria-hidden="true" tabindex="-1"></a> <span class="dt">"is_self"</span><span class="fu">:</span> <span class="kw">true</span></span>
|
||||||
|
<span id="cb7-14"><a href="#cb7-14" aria-hidden="true" tabindex="-1"></a> <span class="fu">}</span><span class="ot">,</span></span>
|
||||||
|
<span id="cb7-15"><a href="#cb7-15" aria-hidden="true" tabindex="-1"></a> <span class="fu">{</span></span>
|
||||||
|
<span id="cb7-16"><a href="#cb7-16" aria-hidden="true" tabindex="-1"></a> <span class="dt">"node_id"</span><span class="fu">:</span> <span class="st">"node-beta"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-17"><a href="#cb7-17" aria-hidden="true" tabindex="-1"></a> <span class="dt">"state"</span><span class="fu">:</span> <span class="st">"alive"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-18"><a href="#cb7-18" aria-hidden="true" tabindex="-1"></a> <span class="dt">"addr"</span><span class="fu">:</span> <span class="st">"203.0.113.11:9400"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-19"><a href="#cb7-19" aria-hidden="true" tabindex="-1"></a> <span class="dt">"region"</span><span class="fu">:</span> <span class="st">"us-east-1"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-20"><a href="#cb7-20" aria-hidden="true" tabindex="-1"></a> <span class="dt">"capabilities"</span><span class="fu">:</span> <span class="ot">[</span><span class="st">"high-memory"</span><span class="ot">]</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-21"><a href="#cb7-21" aria-hidden="true" tabindex="-1"></a> <span class="dt">"load_ratio"</span><span class="fu">:</span> <span class="fl">0.12</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-22"><a href="#cb7-22" aria-hidden="true" tabindex="-1"></a> <span class="dt">"latency_ms"</span><span class="fu">:</span> <span class="dv">2</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-23"><a href="#cb7-23" aria-hidden="true" tabindex="-1"></a> <span class="dt">"last_seen"</span><span class="fu">:</span> <span class="st">"2026-03-29T12:34:55Z"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-24"><a href="#cb7-24" aria-hidden="true" tabindex="-1"></a> <span class="dt">"incarnation"</span><span class="fu">:</span> <span class="dv">3</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb7-25"><a href="#cb7-25" aria-hidden="true" tabindex="-1"></a> <span class="dt">"is_self"</span><span class="fu">:</span> <span class="kw">false</span></span>
|
||||||
|
<span id="cb7-26"><a href="#cb7-26" aria-hidden="true" tabindex="-1"></a> <span class="fu">}</span></span>
|
||||||
|
<span id="cb7-27"><a href="#cb7-27" aria-hidden="true" tabindex="-1"></a> <span class="ot">]</span></span>
|
||||||
|
<span id="cb7-28"><a href="#cb7-28" aria-hidden="true" tabindex="-1"></a><span class="fu">}</span></span></code></pre></div>
|
||||||
|
<h3 id="post-apimeshnodesnode_iddrain">POST
|
||||||
|
/api/mesh/nodes/{node_id}/drain</h3>
|
||||||
|
<p>Puts a node into drain mode (stops accepting new routed tasks,
|
||||||
|
finishes existing ones). Useful before maintenance.</p>
|
||||||
|
<p><strong>Response:</strong></p>
|
||||||
|
<div class="sourceCode" id="cb8"><pre
|
||||||
|
class="sourceCode json"><code class="sourceCode json"><span id="cb8-1"><a href="#cb8-1" aria-hidden="true" tabindex="-1"></a><span class="fu">{</span></span>
|
||||||
|
<span id="cb8-2"><a href="#cb8-2" aria-hidden="true" tabindex="-1"></a> <span class="dt">"node_id"</span><span class="fu">:</span> <span class="st">"node-beta"</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb8-3"><a href="#cb8-3" aria-hidden="true" tabindex="-1"></a> <span class="dt">"drained"</span><span class="fu">:</span> <span class="kw">true</span><span class="fu">,</span></span>
|
||||||
|
<span id="cb8-4"><a href="#cb8-4" aria-hidden="true" tabindex="-1"></a> <span class="dt">"remaining_tasks"</span><span class="fu">:</span> <span class="dv">2</span></span>
|
||||||
|
<span id="cb8-5"><a href="#cb8-5" aria-hidden="true" tabindex="-1"></a><span class="fu">}</span></span></code></pre></div>
|
||||||
|
<h2 id="quick-start">Quick Start</h2>
|
||||||
|
<h3 id="running-two-nodes-on-the-same-machine">Running Two Nodes on the
|
||||||
|
Same Machine</h3>
|
||||||
|
<p><strong>Terminal 1 (Node A):</strong></p>
|
||||||
|
<div class="sourceCode" id="cb9"><pre
|
||||||
|
class="sourceCode bash"><code class="sourceCode bash"><span id="cb9-1"><a href="#cb9-1" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">CLUSTER_ENABLED</span><span class="op">=</span>true</span>
|
||||||
|
<span id="cb9-2"><a href="#cb9-2" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">CLUSTER_SECRET</span><span class="op">=</span><span class="st">"change-me-to-a-strong-shared-secret-at-least-32-characters"</span></span>
|
||||||
|
<span id="cb9-3"><a href="#cb9-3" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">CLUSTER_NODE_ID</span><span class="op">=</span>node-a</span>
|
||||||
|
<span id="cb9-4"><a href="#cb9-4" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">CLUSTER_BIND_PORT</span><span class="op">=</span>9400</span>
|
||||||
|
<span id="cb9-5"><a href="#cb9-5" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">CLUSTER_BEACON_PORT</span><span class="op">=</span>9401</span>
|
||||||
|
<span id="cb9-6"><a href="#cb9-6" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">DATABASE_URL</span><span class="op">=</span>postgres://localhost/optimclaw_a</span>
|
||||||
|
<span id="cb9-7"><a href="#cb9-7" aria-hidden="true" tabindex="-1"></a></span>
|
||||||
|
<span id="cb9-8"><a href="#cb9-8" aria-hidden="true" tabindex="-1"></a><span class="ex">optimclaw</span> onboard <span class="co"># if not already configured</span></span>
|
||||||
|
<span id="cb9-9"><a href="#cb9-9" aria-hidden="true" tabindex="-1"></a><span class="ex">cargo</span> run</span></code></pre></div>
|
||||||
|
<p><strong>Terminal 2 (Node B):</strong></p>
|
||||||
|
<div class="sourceCode" id="cb10"><pre
|
||||||
|
class="sourceCode bash"><code class="sourceCode bash"><span id="cb10-1"><a href="#cb10-1" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">CLUSTER_ENABLED</span><span class="op">=</span>true</span>
|
||||||
|
<span id="cb10-2"><a href="#cb10-2" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">CLUSTER_SECRET</span><span class="op">=</span><span class="st">"change-me-to-a-strong-shared-secret-at-least-32-characters"</span></span>
|
||||||
|
<span id="cb10-3"><a href="#cb10-3" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">CLUSTER_NODE_ID</span><span class="op">=</span>node-b</span>
|
||||||
|
<span id="cb10-4"><a href="#cb10-4" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">CLUSTER_BIND_PORT</span><span class="op">=</span>9410</span>
|
||||||
|
<span id="cb10-5"><a href="#cb10-5" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">CLUSTER_BEACON_PORT</span><span class="op">=</span>9401 <span class="co"># same beacon port so they discover each other</span></span>
|
||||||
|
<span id="cb10-6"><a href="#cb10-6" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">DATABASE_URL</span><span class="op">=</span>postgres://localhost/optimclaw_b</span>
|
||||||
|
<span id="cb10-7"><a href="#cb10-7" aria-hidden="true" tabindex="-1"></a></span>
|
||||||
|
<span id="cb10-8"><a href="#cb10-8" aria-hidden="true" tabindex="-1"></a><span class="ex">optimclaw</span> onboard</span>
|
||||||
|
<span id="cb10-9"><a href="#cb10-9" aria-hidden="true" tabindex="-1"></a><span class="ex">cargo</span> run</span></code></pre></div>
|
||||||
|
<p>Within 5 seconds, both nodes should discover each other via UDP
|
||||||
|
beacons. Verify by hitting the status endpoint:</p>
|
||||||
|
<div class="sourceCode" id="cb11"><pre
|
||||||
|
class="sourceCode bash"><code class="sourceCode bash"><span id="cb11-1"><a href="#cb11-1" aria-hidden="true" tabindex="-1"></a><span class="ex">curl</span> http://localhost:3000/api/mesh/status <span class="kw">|</span> <span class="ex">jq</span> .membership</span>
|
||||||
|
<span id="cb11-2"><a href="#cb11-2" aria-hidden="true" tabindex="-1"></a><span class="co"># {"alive": 2, "suspect": 0, "dead": 0, "total_seen": 2}</span></span></code></pre></div>
|
||||||
|
<h3 id="running-across-machines">Running Across Machines</h3>
|
||||||
|
<p>On each machine, set the same <code>CLUSTER_SECRET</code> and
|
||||||
|
either:</p>
|
||||||
|
<ul>
|
||||||
|
<li>Ensure UDP broadcast works on the local network (same subnet),
|
||||||
|
or</li>
|
||||||
|
<li>Set <code>CLUSTER_SEEDS</code> to the address of at least one other
|
||||||
|
node:</li>
|
||||||
|
</ul>
|
||||||
|
<div class="sourceCode" id="cb12"><pre
|
||||||
|
class="sourceCode bash"><code class="sourceCode bash"><span id="cb12-1"><a href="#cb12-1" aria-hidden="true" tabindex="-1"></a><span class="bu">export</span> <span class="va">CLUSTER_SEEDS</span><span class="op">=</span>192.168.1.100:9400</span></code></pre></div>
|
||||||
|
<h2 id="security-model">Security Model</h2>
|
||||||
|
<h3 id="threat-model">Threat Model</h3>
|
||||||
|
<p>The mesh cluster is designed to be secure against:</p>
|
||||||
|
<ol type="1">
|
||||||
|
<li><strong>Passive eavesdropping</strong> – All traffic is encrypted
|
||||||
|
with AES-256-GCM.</li>
|
||||||
|
<li><strong>Active MITM</strong> – The ML-KEM-768 handshake is bound to
|
||||||
|
<code>CLUSTER_SECRET</code>, preventing interception by parties without
|
||||||
|
the pre-shared key.</li>
|
||||||
|
<li><strong>Quantum adversaries</strong> – ML-KEM-768 provides
|
||||||
|
post-quantum security for key exchange.</li>
|
||||||
|
<li><strong>Rogue node injection</strong> – Beacons are authenticated
|
||||||
|
with HMAC-SHA256; the handshake requires
|
||||||
|
<code>CLUSTER_SECRET</code>.</li>
|
||||||
|
<li><strong>Replay attacks</strong> – Nonces are strictly monotonic;
|
||||||
|
replayed messages are rejected.</li>
|
||||||
|
<li><strong>Partition exploitation</strong> – Nodes degrade to
|
||||||
|
independent operation; no split-brain data corruption.</li>
|
||||||
|
</ol>
|
||||||
|
<h3 id="trust-boundaries">Trust Boundaries</h3>
|
||||||
|
<ul>
|
||||||
|
<li>All nodes sharing the same <code>CLUSTER_SECRET</code> are in the
|
||||||
|
same trust domain.</li>
|
||||||
|
<li>A compromised <code>CLUSTER_SECRET</code> means any attacker can
|
||||||
|
join the cluster. Rotate the secret and restart all nodes if a
|
||||||
|
compromise is suspected.</li>
|
||||||
|
<li>Task data (including tool outputs) is encrypted in transit but
|
||||||
|
available in plaintext to any node in the cluster. Do not add untrusted
|
||||||
|
machines to a cluster that handles sensitive data.</li>
|
||||||
|
</ul>
|
||||||
|
<h3 id="network-recommendations">Network Recommendations</h3>
|
||||||
|
<table>
|
||||||
|
<colgroup>
|
||||||
|
<col style="width: 42%" />
|
||||||
|
<col style="width: 57%" />
|
||||||
|
</colgroup>
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>Deployment</th>
|
||||||
|
<th>Recommendation</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td>Same LAN</td>
|
||||||
|
<td>UDP beacons work out of the box. Use a firewall to restrict beacon
|
||||||
|
and transport ports to trusted hosts.</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Cross-datacenter</td>
|
||||||
|
<td>Use <code>CLUSTER_SEEDS</code>, disable beacons by setting
|
||||||
|
<code>CLUSTER_BEACON_INTERVAL_MS=0</code>, enable TLS
|
||||||
|
(<code>CLUSTER_TLS_CERT</code> / <code>CLUSTER_TLS_KEY</code>), and
|
||||||
|
restrict access via network ACLs.</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td>Cloud (AWS/GCP/Azure)</td>
|
||||||
|
<td>Use private VPC networking. Set <code>CLUSTER_ADVERTISE_ADDR</code>
|
||||||
|
to the private IP. Use security groups to restrict ports 9400-9401.</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
<h2 id="troubleshooting">Troubleshooting</h2>
|
||||||
|
<h3 id="nodes-not-discovering-each-other">Nodes not discovering each
|
||||||
|
other</h3>
|
||||||
|
<ol type="1">
|
||||||
|
<li><strong>Check <code>CLUSTER_SECRET</code></strong> – Must be
|
||||||
|
identical on all nodes. Even trailing whitespace matters.</li>
|
||||||
|
<li><strong>Check beacon port</strong> – All nodes must use the same
|
||||||
|
<code>CLUSTER_BEACON_PORT</code>.</li>
|
||||||
|
<li><strong>Check UDP broadcast</strong> – Some cloud providers and
|
||||||
|
corporate networks block UDP broadcast. Use <code>CLUSTER_SEEDS</code>
|
||||||
|
instead.</li>
|
||||||
|
<li><strong>Check firewall</strong> – Ports
|
||||||
|
<code>CLUSTER_BEACON_PORT</code> (UDP) and
|
||||||
|
<code>CLUSTER_BIND_PORT</code> (TCP) must be open.</li>
|
||||||
|
<li><strong>Check logs</strong> – Run with
|
||||||
|
<code>RUST_LOG=optimclaw::cluster=debug</code> to see beacon
|
||||||
|
send/receive events.</li>
|
||||||
|
</ol>
|
||||||
|
<h3 id="node-stuck-in-suspect-state">Node stuck in Suspect state</h3>
|
||||||
|
<ul>
|
||||||
|
<li>This typically means the node is slow to respond to probes.</li>
|
||||||
|
<li>Increase <code>CLUSTER_PROBE_TIMEOUT_MS</code> on busy nodes.</li>
|
||||||
|
<li>Increase <code>CLUSTER_SUSPICION_MULT</code> to give more time
|
||||||
|
before declaring a node dead.</li>
|
||||||
|
<li>Check if the node is CPU-starved or under heavy I/O load.</li>
|
||||||
|
</ul>
|
||||||
|
<h3 id="high-task-routing-latency">High task routing latency</h3>
|
||||||
|
<ul>
|
||||||
|
<li>Check
|
||||||
|
<code>curl localhost:3000/api/mesh/nodes | jq '.nodes[].latency_ms'</code>
|
||||||
|
to identify slow links.</li>
|
||||||
|
<li>Tasks are only routed away from the local node if a remote node
|
||||||
|
scores >10% better. If most tasks should stay local, this is expected
|
||||||
|
behavior.</li>
|
||||||
|
<li>For cross-datacenter deployments, set <code>CLUSTER_REGION</code> on
|
||||||
|
each node so the locality bonus keeps tasks close.</li>
|
||||||
|
</ul>
|
||||||
|
<h3 id="session-key-negotiation-failures">Session key negotiation
|
||||||
|
failures</h3>
|
||||||
|
<ul>
|
||||||
|
<li>Both nodes must support the same protocol version. Ensure all nodes
|
||||||
|
are running the same OptimClaw release.</li>
|
||||||
|
<li>If using TLS (<code>CLUSTER_TLS_CERT</code>), verify the certificate
|
||||||
|
is valid and trusted by the other node.</li>
|
||||||
|
<li>Check for clock skew greater than 5 minutes between nodes.</li>
|
||||||
|
</ul>
|
||||||
|
<h3 id="node-rejoining-after-network-partition">Node rejoining after
|
||||||
|
network partition</h3>
|
||||||
|
<ul>
|
||||||
|
<li>After a partition heals, the previously-dead node sends beacons
|
||||||
|
again and is rediscovered.</li>
|
||||||
|
<li>The rejoining node increments its incarnation number to override the
|
||||||
|
Dead state in other nodes’ membership lists.</li>
|
||||||
|
<li>Any tasks that were in-flight to the partitioned node will have
|
||||||
|
timed out and been retried locally.</li>
|
||||||
|
</ul>
|
||||||
|
<h3 id="diagnostic-commands">Diagnostic Commands</h3>
|
||||||
|
<div class="sourceCode" id="cb13"><pre
|
||||||
|
class="sourceCode bash"><code class="sourceCode bash"><span id="cb13-1"><a href="#cb13-1" aria-hidden="true" tabindex="-1"></a><span class="co"># Check cluster status</span></span>
|
||||||
|
<span id="cb13-2"><a href="#cb13-2" aria-hidden="true" tabindex="-1"></a><span class="ex">curl</span> <span class="at">-s</span> http://localhost:3000/api/mesh/status <span class="kw">|</span> <span class="ex">jq</span> .</span>
|
||||||
|
<span id="cb13-3"><a href="#cb13-3" aria-hidden="true" tabindex="-1"></a></span>
|
||||||
|
<span id="cb13-4"><a href="#cb13-4" aria-hidden="true" tabindex="-1"></a><span class="co"># List all known nodes</span></span>
|
||||||
|
<span id="cb13-5"><a href="#cb13-5" aria-hidden="true" tabindex="-1"></a><span class="ex">curl</span> <span class="at">-s</span> http://localhost:3000/api/mesh/nodes <span class="kw">|</span> <span class="ex">jq</span> .</span>
|
||||||
|
<span id="cb13-6"><a href="#cb13-6" aria-hidden="true" tabindex="-1"></a></span>
|
||||||
|
<span id="cb13-7"><a href="#cb13-7" aria-hidden="true" tabindex="-1"></a><span class="co"># Drain a node before maintenance</span></span>
|
||||||
|
<span id="cb13-8"><a href="#cb13-8" aria-hidden="true" tabindex="-1"></a><span class="ex">curl</span> <span class="at">-s</span> <span class="at">-X</span> POST http://localhost:3000/api/mesh/nodes/node-beta/drain <span class="kw">|</span> <span class="ex">jq</span> .</span>
|
||||||
|
<span id="cb13-9"><a href="#cb13-9" aria-hidden="true" tabindex="-1"></a></span>
|
||||||
|
<span id="cb13-10"><a href="#cb13-10" aria-hidden="true" tabindex="-1"></a><span class="co"># Watch cluster events in real time</span></span>
|
||||||
|
<span id="cb13-11"><a href="#cb13-11" aria-hidden="true" tabindex="-1"></a><span class="va">RUST_LOG</span><span class="op">=</span>optimclaw::cluster=debug <span class="ex">cargo</span> run <span class="dv">2</span><span class="op">>&</span><span class="dv">1</span> <span class="kw">|</span> <span class="fu">grep</span> cluster</span></code></pre></div>
|
||||||
|
</body>
|
||||||
|
</html>
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 700 480" font-family="Inter, system-ui, sans-serif">
|
||||||
|
<defs>
|
||||||
|
<linearGradient id="bg4" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||||
|
<stop offset="0%" style="stop-color:#1a1a2e"/>
|
||||||
|
<stop offset="100%" style="stop-color:#16213e"/>
|
||||||
|
</linearGradient>
|
||||||
|
</defs>
|
||||||
|
|
||||||
|
<rect width="700" height="480" fill="url(#bg4)" rx="12"/>
|
||||||
|
|
||||||
|
<text x="350" y="35" text-anchor="middle" fill="#00d4aa" font-size="20" font-weight="700">Post-Quantum Handshake Protocol</text>
|
||||||
|
<text x="350" y="55" text-anchor="middle" fill="#8899aa" font-size="11">ML-KEM-768 (Kyber) Key Encapsulation + AES-256-GCM Channel Encryption</text>
|
||||||
|
|
||||||
|
<!-- Node A -->
|
||||||
|
<rect x="60" y="75" width="120" height="35" rx="6" fill="#0f3460" stroke="#5c9cf5" stroke-width="2"/>
|
||||||
|
<text x="120" y="97" text-anchor="middle" fill="#5c9cf5" font-size="13" font-weight="600">Node A (init)</text>
|
||||||
|
<line x1="120" y1="110" x2="120" y2="440" stroke="#5c9cf5" stroke-width="1.5" stroke-opacity="0.3"/>
|
||||||
|
|
||||||
|
<!-- Node B -->
|
||||||
|
<rect x="520" y="75" width="120" height="35" rx="6" fill="#0f3460" stroke="#fab283" stroke-width="2"/>
|
||||||
|
<text x="580" y="97" text-anchor="middle" fill="#fab283" font-size="13" font-weight="600">Node B (resp)</text>
|
||||||
|
<line x1="580" y1="110" x2="580" y2="440" stroke="#fab283" stroke-width="1.5" stroke-opacity="0.3"/>
|
||||||
|
|
||||||
|
<!-- Step 1 -->
|
||||||
|
<g>
|
||||||
|
<rect x="30" y="120" width="170" height="40" rx="4" fill="#0f3460" stroke="#333"/>
|
||||||
|
<text x="115" y="136" text-anchor="middle" fill="#ccc" font-size="9">Generate ML-KEM-768</text>
|
||||||
|
<text x="115" y="150" text-anchor="middle" fill="#00d4aa" font-size="9" font-weight="600">keypair (pk, sk)</text>
|
||||||
|
</g>
|
||||||
|
|
||||||
|
<!-- Arrow 1: A -> B -->
|
||||||
|
<line x1="200" y1="175" x2="520" y2="195" stroke="#5c9cf5" stroke-width="2"/>
|
||||||
|
<polygon points="515,192 525,195 515,198" fill="#5c9cf5"/>
|
||||||
|
<rect x="250" y="170" width="200" height="30" rx="4" fill="#16213e" stroke="#5c9cf5" stroke-width="1"/>
|
||||||
|
<text x="350" y="185" text-anchor="middle" fill="#5c9cf5" font-size="10">{node_id, kem_pk, sign_pk}</text>
|
||||||
|
<text x="350" y="196" text-anchor="middle" fill="#6a6a6a" font-size="8">WebSocket message 1</text>
|
||||||
|
|
||||||
|
<!-- Step 2: B encapsulates -->
|
||||||
|
<g>
|
||||||
|
<rect x="500" y="210" width="180" height="50" rx="4" fill="#0f3460" stroke="#333"/>
|
||||||
|
<text x="590" y="226" text-anchor="middle" fill="#ccc" font-size="9">Encapsulate with A's pk:</text>
|
||||||
|
<text x="590" y="240" text-anchor="middle" fill="#00d4aa" font-size="9" font-weight="600">(shared_secret, ct) =</text>
|
||||||
|
<text x="590" y="252" text-anchor="middle" fill="#00d4aa" font-size="9" font-weight="600">KEM.Encaps(pk_A)</text>
|
||||||
|
</g>
|
||||||
|
|
||||||
|
<!-- Arrow 2: B -> A -->
|
||||||
|
<line x1="520" y1="275" x2="200" y2="295" stroke="#fab283" stroke-width="2"/>
|
||||||
|
<polygon points="205,292 195,295 205,298" fill="#fab283"/>
|
||||||
|
<rect x="220" y="270" width="280" height="30" rx="4" fill="#16213e" stroke="#fab283" stroke-width="1"/>
|
||||||
|
<text x="360" y="285" text-anchor="middle" fill="#fab283" font-size="10">{node_id, ciphertext, sign_pk, ed25519_sig}</text>
|
||||||
|
<text x="360" y="296" text-anchor="middle" fill="#6a6a6a" font-size="8">WebSocket message 2</text>
|
||||||
|
|
||||||
|
<!-- Step 3: A decapsulates -->
|
||||||
|
<g>
|
||||||
|
<rect x="20" y="310" width="190" height="50" rx="4" fill="#0f3460" stroke="#333"/>
|
||||||
|
<text x="115" y="326" text-anchor="middle" fill="#ccc" font-size="9">Verify sig, then decapsulate:</text>
|
||||||
|
<text x="115" y="340" text-anchor="middle" fill="#00d4aa" font-size="9" font-weight="600">shared_secret =</text>
|
||||||
|
<text x="115" y="352" text-anchor="middle" fill="#00d4aa" font-size="9" font-weight="600">KEM.Decaps(ct, sk_A)</text>
|
||||||
|
</g>
|
||||||
|
|
||||||
|
<!-- Step 4: Both derive keys -->
|
||||||
|
<rect x="30" y="375" width="170" height="35" rx="4" fill="#0f3460" stroke="#7fd88f" stroke-width="1.5"/>
|
||||||
|
<text x="115" y="392" text-anchor="middle" fill="#7fd88f" font-size="9" font-weight="600">HKDF-SHA256 → AES keys</text>
|
||||||
|
<text x="115" y="404" text-anchor="middle" fill="#6a6a6a" font-size="8">(send_key, recv_key)</text>
|
||||||
|
|
||||||
|
<rect x="500" y="375" width="180" height="35" rx="4" fill="#0f3460" stroke="#7fd88f" stroke-width="1.5"/>
|
||||||
|
<text x="590" y="392" text-anchor="middle" fill="#7fd88f" font-size="9" font-weight="600">HKDF-SHA256 → AES keys</text>
|
||||||
|
<text x="590" y="404" text-anchor="middle" fill="#6a6a6a" font-size="8">(send_key, recv_key)</text>
|
||||||
|
|
||||||
|
<!-- Encrypted channel -->
|
||||||
|
<rect x="150" y="425" width="400" height="35" rx="6" fill="#303A30" stroke="#7fd88f" stroke-width="2"/>
|
||||||
|
<text x="350" y="447" text-anchor="middle" fill="#7fd88f" font-size="12" font-weight="700">AES-256-GCM Encrypted Channel Active</text>
|
||||||
|
<text x="350" y="455" text-anchor="middle" fill="#6a6a6a" font-size="8">nonce(8B) || ciphertext per frame | forward-secure</text>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 4.6 KiB |
@@ -0,0 +1,56 @@
|
|||||||
|
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 800 500" font-family="Inter, system-ui, sans-serif">
|
||||||
|
<defs>
|
||||||
|
<linearGradient id="bg2" x1="0%" y1="0%" x2="100%" y2="100%">
|
||||||
|
<stop offset="0%" style="stop-color:#1a1a2e"/>
|
||||||
|
<stop offset="100%" style="stop-color:#16213e"/>
|
||||||
|
</linearGradient>
|
||||||
|
</defs>
|
||||||
|
|
||||||
|
<rect width="800" height="500" fill="url(#bg2)" rx="12"/>
|
||||||
|
|
||||||
|
<text x="400" y="40" text-anchor="middle" fill="#e94560" font-size="22" font-weight="700">Task Routing Algorithm</text>
|
||||||
|
<text x="400" y="60" text-anchor="middle" fill="#8899aa" font-size="11">Intelligent capability-based task placement across the mesh</text>
|
||||||
|
|
||||||
|
<!-- Flow diagram -->
|
||||||
|
<!-- Step 1: Task arrives -->
|
||||||
|
<rect x="300" y="80" width="200" height="45" rx="8" fill="#0f3460" stroke="#fab283" stroke-width="2"/>
|
||||||
|
<text x="400" y="107" text-anchor="middle" fill="#fab283" font-size="13" font-weight="600">Task Arrives</text>
|
||||||
|
|
||||||
|
<line x1="400" y1="125" x2="400" y2="150" stroke="#555" stroke-width="2" marker-end="url(#arrow)"/>
|
||||||
|
|
||||||
|
<!-- Step 2: Hard filters -->
|
||||||
|
<rect x="250" y="150" width="300" height="60" rx="8" fill="#0f3460" stroke="#e94560" stroke-width="2"/>
|
||||||
|
<text x="400" y="172" text-anchor="middle" fill="#e94560" font-size="12" font-weight="600">Hard Filters (Pass/Fail)</text>
|
||||||
|
<text x="400" y="190" text-anchor="middle" fill="#8899aa" font-size="10">Required model? Required tools? Min VRAM?</text>
|
||||||
|
<text x="400" y="203" text-anchor="middle" fill="#e06c75" font-size="10">Nodes failing any filter are excluded</text>
|
||||||
|
|
||||||
|
<line x1="400" y1="210" x2="400" y2="235" stroke="#555" stroke-width="2"/>
|
||||||
|
|
||||||
|
<!-- Step 3: Scoring -->
|
||||||
|
<rect x="200" y="235" width="400" height="110" rx="8" fill="#0f3460" stroke="#00d4aa" stroke-width="2"/>
|
||||||
|
<text x="400" y="258" text-anchor="middle" fill="#00d4aa" font-size="13" font-weight="600">Soft Scoring (0.0 - 1.0)</text>
|
||||||
|
|
||||||
|
<g fill="#ccc" font-size="11" font-family="monospace">
|
||||||
|
<text x="230" y="280" fill="#fab283">35%</text><text x="270" y="280">(1.0 - load)</text><text x="440" y="280" fill="#7fd88f">prefer idle nodes</text>
|
||||||
|
<text x="230" y="298" fill="#fab283">25%</text><text x="270" y="298">(vram_free / vram_total)</text><text x="440" y="298" fill="#7fd88f">prefer GPU headroom</text>
|
||||||
|
<text x="230" y="316" fill="#fab283">15%</text><text x="270" y="316">(mem_free / mem_total)</text><text x="440" y="316" fill="#7fd88f">prefer RAM headroom</text>
|
||||||
|
<text x="230" y="334" fill="#fab283">15%</text><text x="270" y="334">(1.0 / hop_count)</text><text x="440" y="334" fill="#7fd88f">prefer closer nodes</text>
|
||||||
|
<text x="230" y="334" fill="#fab283">10%</text><text x="270" y="334">(model_match)</text><text x="440" y="334" fill="#7fd88f">prefer exact match</text>
|
||||||
|
</g>
|
||||||
|
|
||||||
|
<line x1="400" y1="345" x2="400" y2="370" stroke="#555" stroke-width="2"/>
|
||||||
|
|
||||||
|
<!-- Step 4: Selection -->
|
||||||
|
<rect x="275" y="370" width="250" height="45" rx="8" fill="#0f3460" stroke="#5c9cf5" stroke-width="2"/>
|
||||||
|
<text x="400" y="397" text-anchor="middle" fill="#5c9cf5" font-size="13" font-weight="600">Select Highest Score</text>
|
||||||
|
|
||||||
|
<!-- Fork: local vs remote -->
|
||||||
|
<line x1="340" y1="415" x2="250" y2="450" stroke="#7fd88f" stroke-width="2"/>
|
||||||
|
<line x1="460" y1="415" x2="550" y2="450" stroke="#fab283" stroke-width="2"/>
|
||||||
|
|
||||||
|
<rect x="150" y="445" width="160" height="35" rx="6" fill="#303A30" stroke="#7fd88f" stroke-width="1.5"/>
|
||||||
|
<text x="230" y="467" text-anchor="middle" fill="#7fd88f" font-size="12" font-weight="600">Execute Locally</text>
|
||||||
|
|
||||||
|
<rect x="470" y="445" width="200" height="35" rx="6" fill="#3A3030" stroke="#fab283" stroke-width="1.5"/>
|
||||||
|
<text x="570" y="467" text-anchor="middle" fill="#fab283" font-size="12" font-weight="600">Route to Remote Peer</text>
|
||||||
|
</svg>
|
||||||
|
After Width: | Height: | Size: 3.6 KiB |
Reference in New Issue
Block a user