Files
optimclaw/src/tools/schema_validator.rs
T

952 lines
34 KiB
Rust

// === QA Plan P0 - 1.1: Tool schema validator ===
//!
//! Validates tool parameter schemas against OpenAI strict-mode rules.
//!
//! This module provides a comprehensive validation function and a test that
//! exercises every built-in tool's `parameters_schema()` to ensure compatibility
//! with the OpenAI function calling API strict mode.
/// Strict CI-time validation of a JSON schema against OpenAI strict-mode rules.
///
/// Use this function in tests and CI to catch subtle schema defects that the
/// lenient runtime validator allows (freeform properties, missing
/// `additionalProperties`, enum-type mismatches).
///
/// For the lenient runtime variant used at tool-registration time, see
/// [`validate_tool_schema`](crate::tools::tool::validate_tool_schema) in
/// `tool.rs`.
///
/// Returns `Ok(())` if the schema is valid, or `Err(errors)` with a list of
/// all violations found. The validation is recursive for nested objects and
/// array items.
///
/// # Rules enforced
///
/// 1. Top-level must have `"type": "object"`
/// 2. Must have `"properties"` as a JSON object
/// 3. Every key in `"required"` must exist in `"properties"`
/// 4. Every property must have a `"type"` field (freeform/any-type is flagged)
/// 5. `"additionalProperties"` must be explicitly `false` if present
/// 6. Nested objects follow the same rules recursively
/// 7. `"enum"` values must match the declared type
/// 8. Array properties must have an `"items"` definition
pub fn validate_strict_schema(
schema: &serde_json::Value,
tool_name: &str,
) -> Result<(), Vec<String>> {
let errors = check_object_schema(schema, tool_name);
if errors.is_empty() {
Ok(())
} else {
Err(errors)
}
}
/// Recursively validate an object-typed schema node.
fn check_object_schema(schema: &serde_json::Value, path: &str) -> Vec<String> {
let mut errors = Vec::new();
// Rule 1: must have "type": "object"
match schema.get("type").and_then(|t| t.as_str()) {
Some("object") => {}
Some(other) => {
errors.push(format!("{path}: expected type \"object\", got \"{other}\""));
return errors;
}
None => {
errors.push(format!("{path}: missing \"type\": \"object\""));
return errors;
}
}
// Rule 2: must have "properties" as an object
let properties = match schema.get("properties").and_then(|p| p.as_object()) {
Some(p) => p,
None => {
errors.push(format!("{path}: missing or non-object \"properties\""));
return errors;
}
};
// Rule 3: every key in "required" must exist in "properties"
if let Some(required) = schema.get("required").and_then(|r| r.as_array()) {
for req in required {
if let Some(key) = req.as_str()
&& !properties.contains_key(key)
{
errors.push(format!(
"{path}: required key \"{key}\" not found in properties"
));
}
}
}
// Rule 4: every property should have a "type" field
for (key, prop) in properties {
let prop_path = format!("{path}.{key}");
if prop.get("type").is_none() {
// Freeform properties (no type) are intentionally allowed in some tools
// (json "data", http "body") for OpenAI compatibility with union types.
// We flag them as warnings but don't treat them as hard errors.
// Uncomment the next line to enforce strict typing:
// errors.push(format!("{prop_path}: property missing \"type\" field"));
continue;
}
let prop_type = prop.get("type").and_then(|t| t.as_str()).unwrap_or("");
// Rule 5: additionalProperties must be false if present
if let Some(additional) = prop.get("additionalProperties")
&& additional != &serde_json::Value::Bool(false)
// Allow additionalProperties with a type schema (e.g. {"type": "string"})
// which is valid in JSON Schema and used by tools like create_job's credentials.
&& additional.get("type").is_none()
{
errors.push(format!(
"{prop_path}: \"additionalProperties\" should be false or a type schema"
));
}
// Rule 7: enum values must match the declared type
if let Some(enum_values) = prop.get("enum").and_then(|e| e.as_array()) {
for (i, val) in enum_values.iter().enumerate() {
let type_matches = match prop_type {
"string" => val.is_string(),
"integer" | "number" => val.is_number(),
"boolean" => val.is_boolean(),
_ => true, // unknown types: skip check
};
if !type_matches {
errors.push(format!(
"{prop_path}: enum[{i}] value {val} does not match declared type \"{prop_type}\""
));
}
}
}
// Rule 6: nested objects follow the same rules
if prop_type == "object" {
// Objects with additionalProperties as a type schema (e.g. credentials map)
// are valid JSON Schema patterns, not strict-mode objects with fixed properties.
if prop.get("additionalProperties").is_some() && prop.get("properties").is_none() {
// This is a map type (e.g. {"type": "object", "additionalProperties": {"type": "string"}})
// Valid pattern, skip recursive object validation.
} else {
errors.extend(check_object_schema(prop, &prop_path));
}
}
// Rule 8: arrays must have "items"
if prop_type == "array" {
if prop.get("items").is_none() {
errors.push(format!("{prop_path}: array property missing \"items\""));
} else if let Some(items) = prop.get("items") {
// Recurse into items if they are objects
if items.get("type").and_then(|t| t.as_str()) == Some("object") {
errors.extend(check_object_schema(items, &format!("{prop_path}.items")));
}
}
}
}
// Also check top-level additionalProperties (rule 5)
if let Some(additional) = schema.get("additionalProperties")
&& additional != &serde_json::Value::Bool(false)
&& additional.get("type").is_none()
{
errors.push(format!(
"{path}: top-level \"additionalProperties\" should be false or a type schema"
));
}
errors
}
#[cfg(test)]
mod tests {
use super::*;
// ── Unit tests for the validator itself ──────────────────────────────
#[test]
fn test_valid_schema_passes() {
let schema = serde_json::json!({
"type": "object",
"properties": {
"name": { "type": "string", "description": "A name" }
},
"required": ["name"]
});
assert!(validate_strict_schema(&schema, "test").is_ok());
}
#[test]
fn test_missing_type_fails() {
let schema = serde_json::json!({
"properties": {
"name": { "type": "string" }
}
});
let err = validate_strict_schema(&schema, "test").unwrap_err();
assert!(err[0].contains("missing \"type\": \"object\""));
}
#[test]
fn test_wrong_type_fails() {
let schema = serde_json::json!({ "type": "string" });
let err = validate_strict_schema(&schema, "test").unwrap_err();
assert!(err[0].contains("expected type \"object\""));
}
#[test]
fn test_required_not_in_properties_fails() {
let schema = serde_json::json!({
"type": "object",
"properties": {
"name": { "type": "string" }
},
"required": ["name", "age"]
});
let err = validate_strict_schema(&schema, "test").unwrap_err();
assert!(err.iter().any(|e| e.contains("\"age\" not found")));
}
#[test]
fn test_nested_object_validated() {
let schema = serde_json::json!({
"type": "object",
"properties": {
"config": {
"type": "object",
"properties": {
"key": { "type": "string" }
},
"required": ["key", "missing"]
}
}
});
let err = validate_strict_schema(&schema, "test").unwrap_err();
assert!(
err.iter()
.any(|e| e.contains("test.config") && e.contains("\"missing\""))
);
}
#[test]
fn test_array_missing_items_fails() {
let schema = serde_json::json!({
"type": "object",
"properties": {
"tags": { "type": "array", "description": "Tags" }
}
});
let err = validate_strict_schema(&schema, "test").unwrap_err();
assert!(
err.iter()
.any(|e| e.contains("array property missing \"items\""))
);
}
#[test]
fn test_array_with_items_passes() {
let schema = serde_json::json!({
"type": "object",
"properties": {
"tags": {
"type": "array",
"items": { "type": "string" }
}
}
});
assert!(validate_strict_schema(&schema, "test").is_ok());
}
#[test]
fn test_enum_type_mismatch_fails() {
let schema = serde_json::json!({
"type": "object",
"properties": {
"mode": {
"type": "string",
"enum": ["fast", 42, "slow"]
}
}
});
let err = validate_strict_schema(&schema, "test").unwrap_err();
assert!(err.iter().any(|e| e.contains("enum[1]")));
}
#[test]
fn test_enum_matching_type_passes() {
let schema = serde_json::json!({
"type": "object",
"properties": {
"mode": {
"type": "string",
"enum": ["fast", "slow"]
}
}
});
assert!(validate_strict_schema(&schema, "test").is_ok());
}
#[test]
fn test_nested_array_items_object_validated() {
let schema = serde_json::json!({
"type": "object",
"properties": {
"headers": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": { "type": "string" }
},
"required": ["name", "ghost"]
}
}
}
});
let err = validate_strict_schema(&schema, "test").unwrap_err();
assert!(
err.iter()
.any(|e| e.contains("headers.items") && e.contains("\"ghost\""))
);
}
#[test]
fn test_additional_properties_false_passes() {
let schema = serde_json::json!({
"type": "object",
"properties": {
"header": {
"type": "object",
"properties": {
"name": { "type": "string" }
},
"additionalProperties": false
}
}
});
assert!(validate_strict_schema(&schema, "test").is_ok());
}
#[test]
fn test_additional_properties_type_schema_passes() {
// Map pattern: {"type": "object", "additionalProperties": {"type": "string"}}
let schema = serde_json::json!({
"type": "object",
"properties": {
"credentials": {
"type": "object",
"description": "Map of secret names to env var names",
"additionalProperties": { "type": "string" }
}
}
});
assert!(validate_strict_schema(&schema, "test").is_ok());
}
// ── Comprehensive test: validate ALL built-in tool schemas ───────────
#[test]
fn test_all_simple_tool_schemas() {
use crate::tools::Tool;
use crate::tools::builtin::{
ApplyPatchTool, EchoTool, HttpTool, JsonTool, ListDirTool, ReadFileTool, ShellTool,
TimeTool, WriteFileTool,
};
let tools: Vec<Box<dyn Tool>> = vec![
Box::new(EchoTool),
Box::new(TimeTool),
Box::new(JsonTool),
Box::new(HttpTool::new()),
Box::new(ShellTool::new()),
Box::new(ReadFileTool::new()),
Box::new(WriteFileTool::new()),
Box::new(ListDirTool::new()),
Box::new(ApplyPatchTool::new()),
];
let mut failures = Vec::new();
for tool in &tools {
let schema = tool.parameters_schema();
if let Err(errors) = validate_strict_schema(&schema, tool.name()) {
failures.push(format!("Tool '{}': {}", tool.name(), errors.join("; ")));
}
}
assert!(
failures.is_empty(),
"Schema validation failures:\n{}",
failures.join("\n")
);
}
#[test]
fn test_job_tool_schemas() {
use std::sync::Arc;
use crate::context::ContextManager;
use crate::tools::Tool;
use crate::tools::builtin::{CancelJobTool, CreateJobTool, JobStatusTool, ListJobsTool};
let ctx_mgr = Arc::new(ContextManager::new(5));
let tools: Vec<Box<dyn Tool>> = vec![
Box::new(CreateJobTool::new(Arc::clone(&ctx_mgr))),
Box::new(ListJobsTool::new(Arc::clone(&ctx_mgr))),
Box::new(JobStatusTool::new(Arc::clone(&ctx_mgr))),
Box::new(CancelJobTool::new(Arc::clone(&ctx_mgr))),
];
let mut failures = Vec::new();
for tool in &tools {
let schema = tool.parameters_schema();
if let Err(errors) = validate_strict_schema(&schema, tool.name()) {
failures.push(format!("Tool '{}': {}", tool.name(), errors.join("; ")));
}
}
assert!(
failures.is_empty(),
"Schema validation failures:\n{}",
failures.join("\n")
);
}
#[test]
fn test_skill_tool_schemas() {
use std::sync::Arc;
use crate::skills::catalog::SkillCatalog;
use crate::skills::registry::SkillRegistry;
use crate::tools::Tool;
use crate::tools::builtin::{
SkillInstallTool, SkillListTool, SkillRemoveTool, SkillSearchTool,
};
let dir = tempfile::tempdir().expect("tempdir");
let path = dir.keep();
let registry = Arc::new(std::sync::RwLock::new(SkillRegistry::new(path)));
let catalog = Arc::new(SkillCatalog::with_url("http://127.0.0.1:1"));
let tools: Vec<Box<dyn Tool>> = vec![
Box::new(SkillListTool::new(Arc::clone(&registry))),
Box::new(SkillSearchTool::new(
Arc::clone(&registry),
Arc::clone(&catalog),
)),
Box::new(SkillInstallTool::new(
Arc::clone(&registry),
Arc::clone(&catalog),
)),
Box::new(SkillRemoveTool::new(Arc::clone(&registry))),
];
let mut failures = Vec::new();
for tool in &tools {
let schema = tool.parameters_schema();
if let Err(errors) = validate_strict_schema(&schema, tool.name()) {
failures.push(format!("Tool '{}': {}", tool.name(), errors.join("; ")));
}
}
assert!(
failures.is_empty(),
"Schema validation failures:\n{}",
failures.join("\n")
);
}
/// Validate schemas from tools that cannot be easily constructed by
/// inlining the JSON schema directly. This covers the extension tools and
/// routine tools whose constructors require heavy dependencies.
#[test]
fn test_inline_schemas_for_complex_tools() {
// These schemas are extracted from the source code of tools with complex deps.
// If the source schemas change, these tests serve as a canary.
let schemas: Vec<(&str, serde_json::Value)> = vec![
// Extension tools
(
"tool_search",
serde_json::json!({
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "Search query"
},
"discover": {
"type": "boolean",
"description": "Search online",
"default": false
}
},
"required": ["query"]
}),
),
(
"tool_install",
serde_json::json!({
"type": "object",
"properties": {
"name": { "type": "string", "description": "Extension name" },
"url": { "type": "string", "description": "Explicit URL" },
"kind": {
"type": "string",
"enum": ["mcp_server", "wasm_tool", "wasm_channel"],
"description": "Extension type"
}
},
"required": ["name"]
}),
),
(
"tool_auth",
serde_json::json!({
"type": "object",
"properties": {
"name": { "type": "string", "description": "Extension name" }
},
"required": ["name"]
}),
),
(
"tool_activate",
serde_json::json!({
"type": "object",
"properties": {
"name": { "type": "string", "description": "Extension name" }
},
"required": ["name"]
}),
),
(
"tool_list",
serde_json::json!({
"type": "object",
"properties": {
"kind": {
"type": "string",
"enum": ["mcp_server", "wasm_tool", "wasm_channel"],
"description": "Filter by extension type"
},
"include_available": {
"type": "boolean",
"description": "Include not-yet-installed entries",
"default": false
}
}
}),
),
(
"tool_remove",
serde_json::json!({
"type": "object",
"properties": {
"name": { "type": "string", "description": "Extension name" }
},
"required": ["name"]
}),
),
// Routine tools
(
"routine_create",
crate::tools::builtin::routine::routine_create_parameters_schema(),
),
(
"routine_list",
serde_json::json!({
"type": "object",
"properties": {},
"required": []
}),
),
(
"routine_update",
crate::tools::builtin::routine::routine_update_parameters_schema(),
),
(
"routine_delete",
serde_json::json!({
"type": "object",
"properties": {
"name": { "type": "string", "description": "Name" }
},
"required": ["name"]
}),
),
(
"routine_fire",
serde_json::json!({
"type": "object",
"properties": {
"name": { "type": "string", "description": "Routine name" }
},
"required": ["name"]
}),
),
(
"routine_history",
serde_json::json!({
"type": "object",
"properties": {
"name": { "type": "string", "description": "Routine name" },
"limit": { "type": "integer", "description": "Max runs", "default": 10 }
},
"required": ["name"]
}),
),
(
"event_emit",
serde_json::json!({
"type": "object",
"properties": {
"event_source": { "type": "string", "description": "Event source" },
"event_type": { "type": "string", "description": "Event type" },
"payload": { "type": "object", "description": "Event payload", "properties": {} }
},
"required": ["event_source", "event_type"]
}),
),
// Job tools with complex deps
(
"job_events",
serde_json::json!({
"type": "object",
"properties": {
"job_id": { "type": "string", "description": "Job ID" },
"limit": { "type": "integer", "description": "Max events" }
},
"required": ["job_id"]
}),
),
(
"job_prompt",
serde_json::json!({
"type": "object",
"properties": {
"job_id": { "type": "string", "description": "Job ID" },
"content": { "type": "string", "description": "Prompt text" },
"done": { "type": "boolean", "description": "Signal finish" }
},
"required": ["job_id", "content"]
}),
),
];
let mut failures = Vec::new();
for (name, schema) in &schemas {
if let Err(errors) = validate_strict_schema(schema, name) {
failures.push(format!("Tool '{}': {}", name, errors.join("; ")));
}
}
assert!(
failures.is_empty(),
"Schema validation failures for inline schemas:\n{}",
failures.join("\n")
);
}
/// Validate that the memory tool schemas (which need Workspace) are correct.
/// Since Workspace requires a database connection, we validate the schemas
/// are structurally correct by inlining them.
#[test]
fn test_memory_tool_schemas_inline() {
let schemas: Vec<(&str, serde_json::Value)> = vec![
(
"memory_search",
serde_json::json!({
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "Search query"
},
"limit": {
"type": "integer",
"description": "Max results",
"default": 5,
"minimum": 1,
"maximum": 20
}
},
"required": ["query"]
}),
),
(
"memory_write",
serde_json::json!({
"type": "object",
"properties": {
"content": { "type": "string", "description": "Content to write" },
"target": { "type": "string", "description": "Where to write", "default": "daily_log" },
"append": { "type": "boolean", "description": "Append or replace", "default": true }
},
"required": ["content"]
}),
),
(
"memory_read",
serde_json::json!({
"type": "object",
"properties": {
"path": { "type": "string", "description": "Path to read" }
},
"required": ["path"]
}),
),
(
"memory_tree",
serde_json::json!({
"type": "object",
"properties": {
"path": { "type": "string", "description": "Root path", "default": "" },
"depth": { "type": "integer", "description": "Max depth", "default": 1, "minimum": 1, "maximum": 10 }
}
}),
),
];
let mut failures = Vec::new();
for (name, schema) in &schemas {
if let Err(errors) = validate_strict_schema(schema, name) {
failures.push(format!("Tool '{}': {}", name, errors.join("; ")));
}
}
assert!(
failures.is_empty(),
"Schema validation failures for memory tool schemas:\n{}",
failures.join("\n")
);
}
// ── WASM and MCP tool schema validation (QA 1.1 extension) ─────────
/// Representative WASM tool schemas -- these mirror the shapes produced by
/// `WasmToolWrapper::parameters_schema()` from real WASM modules.
#[test]
fn test_wasm_tool_schemas() {
let schemas: Vec<(&str, serde_json::Value)> = vec![
// Typical WASM tool with simple params
(
"wasm_weather",
serde_json::json!({
"type": "object",
"properties": {
"city": { "type": "string", "description": "City name" },
"units": {
"type": "string",
"enum": ["celsius", "fahrenheit"],
"description": "Temperature units"
}
},
"required": ["city"]
}),
),
// WASM tool with nested object (e.g., HTTP tool)
(
"wasm_http_client",
serde_json::json!({
"type": "object",
"properties": {
"url": { "type": "string", "description": "URL to fetch" },
"method": {
"type": "string",
"enum": ["GET", "POST", "PUT", "DELETE"],
"description": "HTTP method"
},
"headers": {
"type": "object",
"properties": {},
"description": "Custom headers"
},
"body": { "type": "string", "description": "Request body" }
},
"required": ["url"]
}),
),
// WASM tool with array params
(
"wasm_batch_processor",
serde_json::json!({
"type": "object",
"properties": {
"items": {
"type": "array",
"items": { "type": "string" },
"description": "Items to process"
},
"parallel": { "type": "boolean", "description": "Run in parallel" }
},
"required": ["items"]
}),
),
// Empty WASM tool (no required params)
(
"wasm_status",
serde_json::json!({
"type": "object",
"properties": {}
}),
),
];
let mut failures = Vec::new();
for (name, schema) in &schemas {
if let Err(errors) = validate_strict_schema(schema, name) {
failures.push(format!("WASM tool '{}': {}", name, errors.join("; ")));
}
}
assert!(
failures.is_empty(),
"Schema validation failures for WASM tool schemas:\n{}",
failures.join("\n")
);
}
/// Representative MCP tool schemas -- these mirror the shapes received from
/// MCP servers via `McpTool::input_schema` (camelCase `inputSchema` in protocol).
#[test]
fn test_mcp_tool_schemas() {
let schemas: Vec<(&str, serde_json::Value)> = vec![
// Default MCP schema (empty object -- from default_input_schema())
(
"mcp_default",
serde_json::json!({"type": "object", "properties": {}}),
),
// Typical MCP server tool (e.g., filesystem server)
(
"mcp_read_file",
serde_json::json!({
"type": "object",
"properties": {
"path": { "type": "string", "description": "File path to read" }
},
"required": ["path"]
}),
),
// MCP tool with complex nested params (e.g., database query)
(
"mcp_sql_query",
serde_json::json!({
"type": "object",
"properties": {
"query": { "type": "string", "description": "SQL query to execute" },
"params": {
"type": "array",
"items": { "type": "string" },
"description": "Query parameters"
},
"timeout_ms": {
"type": "integer",
"description": "Query timeout in milliseconds"
}
},
"required": ["query"]
}),
),
// MCP tool with additionalProperties: false (strict server)
(
"mcp_strict_tool",
serde_json::json!({
"type": "object",
"properties": {
"action": {
"type": "string",
"enum": ["start", "stop", "restart"],
"description": "Action to perform"
}
},
"required": ["action"],
"additionalProperties": false
}),
),
];
let mut failures = Vec::new();
for (name, schema) in &schemas {
if let Err(errors) = validate_strict_schema(schema, name) {
failures.push(format!("MCP tool '{}': {}", name, errors.join("; ")));
}
}
assert!(
failures.is_empty(),
"Schema validation failures for MCP tool schemas:\n{}",
failures.join("\n")
);
}
/// Verify the validator catches common issues in externally-sourced schemas.
/// WASM modules and MCP servers may produce schemas with defects that
/// built-in tools wouldn't have.
#[test]
fn test_external_schema_defects_detected() {
// Missing top-level type (MCP server omitted it)
let bad_no_type = serde_json::json!({
"properties": {
"query": { "type": "string" }
}
});
assert!(validate_strict_schema(&bad_no_type, "ext_no_type").is_err());
// Required key not in properties (WASM module typo)
let bad_required = serde_json::json!({
"type": "object",
"properties": {
"input": { "type": "string" }
},
"required": ["inpt"]
});
assert!(validate_strict_schema(&bad_required, "ext_typo").is_err());
// Array without items definition (MCP server bug)
let bad_array = serde_json::json!({
"type": "object",
"properties": {
"tags": { "type": "array" }
}
});
assert!(validate_strict_schema(&bad_array, "ext_no_items").is_err());
// Enum type mismatch (WASM module declares string enum with integers)
let bad_enum = serde_json::json!({
"type": "object",
"properties": {
"mode": {
"type": "string",
"enum": [1, 2, 3]
}
}
});
assert!(validate_strict_schema(&bad_enum, "ext_enum_mismatch").is_err());
// Nested object without type (deeply nested MCP schema)
let bad_nested = serde_json::json!({
"type": "object",
"properties": {
"config": {
"type": "object",
"properties": {
"setting": { "description": "missing type field" }
}
}
}
});
// This may pass or fail depending on whether we enforce type on every
// nested property -- the validator allows freeform for compatibility.
// The important thing is it doesn't panic.
let _ = validate_strict_schema(&bad_nested, "ext_nested_no_type");
}
}