mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-25 14:53:34 +00:00
feat: full image support across all channels (#725)
* feat: full image support across all channels End-to-end image handling: upload, generation, analysis, editing, and rendering across web gateway, HTTP webhook, WASM (Telegram/Slack), and REPL channels. Builds on the attachment infrastructure from #596 and draws inspiration from PR #641's image pipeline approach — credit to that PR's author for the sentinel JSON pattern and base64-in-JSON upload design. Key changes: - Image upload in web UI (file picker, paste, preview strip) - Image generation tool (FLUX/DALL-E via /v1/images/generations) - Image edit tool (multipart /v1/images/edits with fallback) - Image analysis tool (vision model for workspace images) - Model detection utilities (image_models.rs, vision_models.rs) - Sentinel JSON detection in dispatcher for generated image rendering - StatusUpdate::ImageGenerated → SSE/WS/REPL/WASM broadcast - HTTP webhook attachment support (base64, 5MB/file, 10MB total) - WASM channel image download (Telegram via file API, Slack via host HTTP) - Tool registration wiring in app.rs [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: address PR #725 review comments (16 issues) - SecretString for API keys in all image tools (image_gen, image_edit, image_analyze) - Binary image read via tokio::fs::read instead of DB-backed workspace.read() - Replace Arc<Workspace> with Option<PathBuf> base_dir (workspace has no filesystem API) - ApprovalRequirement::UnlessAutoApproved for cost-sensitive image tools - Scope sentinel detection to image_generate/image_edit tool names only - Skip ToolResult preview broadcast for image sentinels (avoids multi-MB base64 in SSE) - Extract shared media_type_from_path() to builtin/mod.rs - Rename fallback_chat_edit → fallback_generate with tracing::warn - Increase gateway body limit from 1MB to 10MB for image uploads - Increase webhook body limit to 15MB (base64 overhead) - Log warning on invalid base64 in images_to_attachments - Client-side image size limits (5MB/file, 5 images max) in app.js - aria-label on attach button for accessibility - Update body_too_large test for new 10MB limit [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: add Slack file size check before download (PR review item #15) Skip downloading files larger than 20 MB in the Slack WASM channel to avoid excessive memory use and slow downloads in the WASM runtime. Logs a warning when a file is skipped. Also bumps channel versions for Slack and Telegram (prior branch changes). [skip-regression-check] Co-Authored-By: Claude Opus 4.6 <[email protected]> * style: cargo fmt Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix(security): add path validation and approval requirement to image tools Add sandbox path validation via validate_path() to both ImageAnalyzeTool and ImageEditTool to prevent path traversal attacks that could exfiltrate arbitrary files through external vision/edit APIs. Also fix ImageAnalyzeTool::requires_approval to return UnlessAutoApproved, consistent with ImageEditTool and ImageGenerateTool. Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: post-download size guards and empty data_url sentinel check - Slack: add post-download size check on actual bytes when metadata size_bytes is absent, preventing bypass of the 20MB limit - Telegram: add 20MB download size limit (matching Slack) enforced in download_telegram_file() after receiving response bytes - Dispatcher: skip broadcasting ImageGenerated SSE event when data_url is empty from unwrap_or_default(), log warning instead Closes correctness issues #3, #4, #5 from PR #725 review. Co-Authored-By: Claude Opus 4.6 <[email protected]> * fix: use mime_guess for media type detection, add alt attrs and media_type validation - Replace hardcoded media type mapping with mime_guess crate (already in deps) - Add alt attributes to img elements in web UI for accessibility - Validate media_type starts with "image/" in images_to_attachments() - Update bmp test assertion to match mime_guess behavior Co-Authored-By: Claude Opus 4.6 <[email protected]> --------- Co-authored-by: Claude Opus 4.6 <[email protected]> Co-authored-by: Zaki <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.6
Zaki
parent
7fb2f47999
commit
553c306c52
Generated
+1
-1
@@ -267,7 +267,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "slack-channel"
|
||||
version = "0.1.0"
|
||||
version = "0.2.1"
|
||||
dependencies = [
|
||||
"hex",
|
||||
"hmac",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[package]
|
||||
name = "slack-channel"
|
||||
version = "0.2.0"
|
||||
version = "0.2.1"
|
||||
edition = "2021"
|
||||
description = "Slack Events API channel for IronClaw"
|
||||
license = "MIT OR Apache-2.0"
|
||||
|
||||
@@ -357,10 +357,108 @@ fn extract_slack_attachments(files: &Option<Vec<SlackFile>>) -> Vec<InboundAttac
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Download a file from Slack using the url_private endpoint.
|
||||
///
|
||||
/// Slack file downloads require Bearer auth with the bot token, which is
|
||||
/// injected by the host credential system via `channel_host::http_request`.
|
||||
fn download_slack_file(url: &str) -> Result<Vec<u8>, String> {
|
||||
let headers = serde_json::json!({});
|
||||
|
||||
let result = channel_host::http_request("GET", url, &headers.to_string(), None, None);
|
||||
|
||||
let response = result.map_err(|e| format!("Slack file download failed: {}", e))?;
|
||||
|
||||
if response.status != 200 {
|
||||
let body_str = String::from_utf8_lossy(&response.body);
|
||||
return Err(format!(
|
||||
"Slack file download returned {}: {}",
|
||||
response.status, body_str
|
||||
));
|
||||
}
|
||||
|
||||
Ok(response.body)
|
||||
}
|
||||
|
||||
/// Download file bytes and store them via the host for processing.
|
||||
///
|
||||
/// Downloads all file types (images, documents, etc.) so the host-side
|
||||
/// middleware can process them (vision pipeline for images, text extraction
|
||||
/// for documents, transcription for audio, etc.).
|
||||
/// Maximum file size to download (20 MB). Files larger than this are skipped
|
||||
/// to avoid excessive memory use and slow downloads in the WASM runtime.
|
||||
const MAX_DOWNLOAD_SIZE_BYTES: u64 = 20 * 1024 * 1024;
|
||||
|
||||
fn download_and_store_slack_files(attachments: &[InboundAttachment]) {
|
||||
for att in attachments {
|
||||
let Some(ref url) = att.source_url else {
|
||||
continue;
|
||||
};
|
||||
|
||||
// Skip files that exceed the size limit
|
||||
if let Some(size) = att.size_bytes {
|
||||
if size > MAX_DOWNLOAD_SIZE_BYTES {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Warn,
|
||||
&format!(
|
||||
"Skipping Slack file download: {} bytes exceeds {} MB limit (id={})",
|
||||
size,
|
||||
MAX_DOWNLOAD_SIZE_BYTES / (1024 * 1024),
|
||||
att.id
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
match download_slack_file(url) {
|
||||
Ok(bytes) => {
|
||||
// Post-download size guard: metadata size_bytes is optional,
|
||||
// so a file with no size info could bypass the pre-download check.
|
||||
if bytes.len() as u64 > MAX_DOWNLOAD_SIZE_BYTES {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Warn,
|
||||
&format!(
|
||||
"Discarding Slack file after download: {} bytes exceeds {} MB limit (id={})",
|
||||
bytes.len(),
|
||||
MAX_DOWNLOAD_SIZE_BYTES / (1024 * 1024),
|
||||
att.id
|
||||
),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!(
|
||||
"Downloaded Slack file: {} bytes, mime={}",
|
||||
bytes.len(),
|
||||
att.mime_type
|
||||
),
|
||||
);
|
||||
if let Err(e) = channel_host::store_attachment_data(&att.id, &bytes) {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Failed to store Slack file data: {}", e),
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Failed to download Slack file: {}", e),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Handle a Slack event and emit message if applicable.
|
||||
fn handle_slack_event(event: SlackEvent, team_id: Option<String>, _event_id: Option<String>) {
|
||||
let attachments = extract_slack_attachments(&event.files);
|
||||
|
||||
// Download and store file attachments for host-side processing
|
||||
download_and_store_slack_files(&attachments);
|
||||
|
||||
match event.event_type.as_str() {
|
||||
// Direct mention of the bot (always in a channel, not a DM)
|
||||
"app_mention" => {
|
||||
@@ -722,4 +820,10 @@ mod tests {
|
||||
let event: SlackEvent = serde_json::from_str(json).unwrap();
|
||||
assert!(event.files.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_max_download_size_constant() {
|
||||
// Verify the constant is 20 MB
|
||||
assert_eq!(MAX_DOWNLOAD_SIZE_BYTES, 20 * 1024 * 1024);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user