diff --git a/.githooks/commit-msg b/.githooks/commit-msg new file mode 120000 index 00000000..2eb95be6 --- /dev/null +++ b/.githooks/commit-msg @@ -0,0 +1 @@ +../scripts/commit-msg-regression.sh \ No newline at end of file diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 00000000..0abd640a --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Pre-commit hook: run version bump checks when WIT or extension sources change. +# Install: git config core.hooksPath .githooks + +# Only run the check if relevant files are staged +STAGED=$(git diff --cached --name-only) + +NEEDS_CHECK=false +if echo "$STAGED" | grep -qE '^wit/|^channels-src/|^tools-src/'; then + NEEDS_CHECK=true +fi + +if $NEEDS_CHECK; then + echo "pre-commit: checking version bumps..." + if ! ./scripts/check-version-bumps.sh; then + echo "" + echo "Commit blocked: version bump check failed." + echo "Bump versions in the relevant registry JSON and/or WIT package declaration." + echo "To bypass: git commit --no-verify" + exit 1 + fi +fi diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_style.yml index 27578570..526c7740 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_style.yml @@ -12,7 +12,6 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - profile: minimal components: rustfmt - name: Check formatting run: cargo fmt --all -- --check @@ -36,7 +35,6 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - profile: minimal components: clippy - uses: Swatinem/rust-cache@v2 with: @@ -63,7 +61,6 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - profile: minimal components: clippy - uses: Swatinem/rust-cache@v2 with: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7c2f564c..8f0fd2bb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,7 +25,6 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - profile: minimal targets: wasm32-wasip2 - uses: Swatinem/rust-cache@v2 with: @@ -45,8 +44,6 @@ jobs: uses: actions/checkout@v6 - name: Install Rust uses: dtolnay/rust-toolchain@stable - with: - profile: minimal - uses: Swatinem/rust-cache@v2 - name: Run Telegram Channel Tests run: cargo test --manifest-path channels-src/telegram/Cargo.toml -- --nocapture @@ -69,8 +66,6 @@ jobs: uses: actions/checkout@v6 - name: Install Rust uses: dtolnay/rust-toolchain@stable - with: - profile: minimal - uses: Swatinem/rust-cache@v2 with: key: windows-${{ matrix.name }} @@ -86,7 +81,6 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@stable with: - profile: minimal targets: wasm32-wasip2 - uses: Swatinem/rust-cache@v2 with: diff --git a/Cargo.lock b/Cargo.lock index 2bf1b890..c6ad733a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "adobe-cmap-parser" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae8abfa9a4688de8fc9f42b3f013b6fffec18ed8a554f5f113577e0b9b3212a3" +dependencies = [ + "pom", +] + [[package]] name = "aead" version = "0.5.2" @@ -176,6 +185,9 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] [[package]] name = "arrayref" @@ -1522,6 +1534,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "derive_more" version = "2.1.1" @@ -1810,6 +1833,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "euclid" +version = "0.20.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bb7ef65b3777a325d1eeefefab5b6d4959da54747e33bd6258e789640f307ad" +dependencies = [ + "num-traits", +] + [[package]] name = "event-listener" version = "5.4.1" @@ -2863,6 +2895,7 @@ dependencies = [ "lru", "mime_guess", "open", + "pdf-extract", "pgvector", "postgres-types", "pretty_assertions", @@ -2910,6 +2943,7 @@ dependencies = [ "wasmtime", "wasmtime-wasi", "zbus", + "zip", ] [[package]] @@ -3251,6 +3285,24 @@ version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lopdf" +version = "0.34.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5c8ecfc6c72051981c0459f75ccc585e7ff67c70829560cda8e647882a9abff" +dependencies = [ + "encoding_rs", + "flate2", + "indexmap 2.13.0", + "itoa", + "log", + "md-5", + "nom", + "rangemap", + "time", + "weezl", +] + [[package]] name = "lru" version = "0.16.3" @@ -3794,6 +3846,21 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" +[[package]] +name = "pdf-extract" +version = "0.7.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbb3a5387b94b9053c1e69d8abfd4dd6dae7afda65a5c5279bc1f42ab39df575" +dependencies = [ + "adobe-cmap-parser", + "encoding_rs", + "euclid", + "lopdf", + "postscript", + "type1-encoding-parser", + "unicode-normalization", +] + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -3993,6 +4060,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "pom" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60f6ce597ecdcc9a098e7fddacb1065093a3d66446fa16c675e7e71d1b5c28e6" + [[package]] name = "postcard" version = "1.1.3" @@ -4038,6 +4111,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "postscript" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78451badbdaebaf17f053fd9152b3ffb33b516104eacb45e7864aaa9c712f306" + [[package]] name = "potential_utf" version = "0.1.4" @@ -4315,6 +4394,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rangemap" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "973443cf09a9c8656b574a866ab68dfa19f0867d0340648c7d2f6a71b8a8ea68" + [[package]] name = "rayon" version = "1.11.0" @@ -6291,6 +6376,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "type1-encoding-parser" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3d6cc09e1a99c7e01f2afe4953789311a1c50baebbdac5b477ecf78e2e92a5b" +dependencies = [ + "pom", +] + [[package]] name = "typenum" version = "1.19.0" @@ -7042,6 +7136,12 @@ dependencies = [ "string_cache_codegen", ] +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "which" version = "4.4.2" @@ -7849,12 +7949,41 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "zip" +version = "2.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap 2.13.0", + "memchr", + "thiserror 2.0.18", + "zopfli", +] + [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.3" diff --git a/Cargo.toml b/Cargo.toml index 3f1e78ae..237717d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,7 @@ tokio-stream = { version = "0.1", features = ["sync"] } futures = "0.3" # HTTP client -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots", "stream"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls-native-roots", "stream"] } # Serialization serde = { version = "1", features = ["derive"] } @@ -147,6 +147,10 @@ bollard = "0.18" flate2 = "1" tar = "0.4" +# Document text extraction +pdf-extract = "0.7" +zip = { version = "2", default-features = false, features = ["deflate"] } + # HTTP proxy for sandboxed network access hyper = { version = "1.5", features = ["server", "http1", "http2"] } hyper-util = { version = "0.1", features = ["server", "tokio", "http1", "http2"] } diff --git a/FEATURE_PARITY.md b/FEATURE_PARITY.md index 71472ec5..359e0b6c 100644 --- a/FEATURE_PARITY.md +++ b/FEATURE_PARITY.md @@ -119,7 +119,7 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Mention-based activation | ✅ | ✅ | bot_username + respond_to_all_group_messages | | Per-group tool policies | ✅ | ❌ | Allow/deny specific tools | | Thread isolation | ✅ | ✅ | Separate sessions per thread | -| Per-channel media limits | ✅ | 🚧 | Caption support for media; no size limits | +| Per-channel media limits | ✅ | ✅ | Attachment type in WIT; max 10 per msg, 20MB total, MIME allowlist | | Typing indicators | ✅ | 🚧 | TUI + Telegram typing/actionable status prompts; richer parity pending | | Per-channel ackReaction config | ✅ | ❌ | Customizable acknowledgement reactions | | Group session priming | ✅ | ❌ | Member roster injected for context | @@ -248,19 +248,32 @@ This document tracks feature parity between IronClaw (Rust implementation) and O | Feature | OpenClaw | IronClaw | Priority | Notes | |---------|----------|----------|----------|-------| +| WIT inbound-attachment type | N/A | ✅ | P1 | `inbound-attachment` record in channel-host (id, mime_type, filename, size_bytes, source_url, storage_key, extracted_text) | +| WIT outbound attachment type | N/A | ✅ | P1 | `attachment` record in channel (filename, mime_type, data) on `agent-response` | +| WIT on-broadcast export | N/A | ✅ | P1 | Proactive message sending without prior incoming message | +| IncomingMessage attachments | N/A | ✅ | P1 | `IncomingAttachment` struct on `IncomingMessage`, populated from WASM channels | +| OutgoingResponse attachments | N/A | ✅ | P1 | File paths on `OutgoingResponse`, read from disk and sent as WIT attachments | +| Attachment security (size/MIME) | N/A | ✅ | P1 | Inbound: max 10, 20MB total, MIME allowlist. Outbound: 50MB total | +| Telegram media parsing | ✅ | ✅ | P1 | Photo, document, audio, video, voice, sticker parsed and emitted as attachments | +| Telegram media sending | ✅ | ✅ | P1 | sendPhoto/sendDocument multipart upload, auto photo→document fallback >10MB | +| Slack file parsing | ✅ | ✅ | P1 | `files` array from Events API parsed into attachments | +| WhatsApp media parsing | ✅ | ✅ | P1 | Image, audio, video, document parsed with caption as extracted_text | +| Discord attachment parsing | ✅ | ❌ | P2 | Discord interaction payloads don't include file attachments (needs message events) | +| HTTP tool save_to | N/A | ✅ | P1 | Download binary files to /tmp/ for attachment sending (50MB limit, path traversal protection) | +| Credential env var fallback | N/A | ✅ | P2 | Channels can use env vars (e.g., TELEGRAM_BOT_TOKEN) when secrets store not configured | | Image processing (Sharp) | ✅ | ❌ | P2 | Resize, format convert | | Configurable image resize dims | ✅ | ❌ | P2 | Per-agent dimension config | | Multiple images per tool call | ✅ | ❌ | P2 | Single tool invocation, multiple images | | Audio transcription | ✅ | ❌ | P2 | | | Video support | ✅ | ❌ | P3 | | | PDF parsing | ✅ | ❌ | P2 | pdfjs-dist | -| MIME detection | ✅ | ❌ | P2 | | +| MIME detection | ✅ | ✅ | P2 | MIME allowlist in host validates attachment types | | Media caching | ✅ | ❌ | P3 | | | Vision model integration | ✅ | ❌ | P2 | Image understanding | | TTS (Edge TTS) | ✅ | ❌ | P3 | Text-to-speech | | TTS (OpenAI) | ✅ | ❌ | P3 | | | Incremental TTS playback | ✅ | ❌ | P3 | iOS progressive playback | -| Sticker-to-image | ✅ | ❌ | P3 | Telegram stickers | +| Sticker-to-image | ✅ | ✅ | P3 | Telegram stickers emitted as image/webp attachments | ### Owner: _Unassigned_ diff --git a/channels-src/discord/Cargo.toml b/channels-src/discord/Cargo.toml index e10072e4..81e95260 100644 --- a/channels-src/discord/Cargo.toml +++ b/channels-src/discord/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "discord-channel" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Discord channel for IronClaw" license = "MIT OR Apache-2.0" diff --git a/channels-src/discord/discord.capabilities.json b/channels-src/discord/discord.capabilities.json index f2d3e69e..fd55c685 100644 --- a/channels-src/discord/discord.capabilities.json +++ b/channels-src/discord/discord.capabilities.json @@ -1,6 +1,6 @@ { - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "type": "channel", "name": "discord", "description": "Discord Gateway/Webhook channel for handling slash commands, buttons, and messages", diff --git a/channels-src/discord/src/lib.rs b/channels-src/discord/src/lib.rs index beb856cd..c8b37428 100644 --- a/channels-src/discord/src/lib.rs +++ b/channels-src/discord/src/lib.rs @@ -312,6 +312,10 @@ impl Guest for DiscordChannel { fn on_status(_update: StatusUpdate) {} + fn on_broadcast(_user_id: String, _response: AgentResponse) -> Result<(), String> { + Err("broadcast not yet implemented for Discord channel".to_string()) + } + fn on_shutdown() { channel_host::log( channel_host::LogLevel::Info, @@ -414,6 +418,7 @@ fn handle_slash_command(interaction: &DiscordInteraction) -> bool { content, thread_id: None, metadata_json, + attachments: vec![], }); true } @@ -467,6 +472,7 @@ fn handle_message_component(interaction: &DiscordInteraction, message: &DiscordM content: format!("[Button clicked] {}", message.content), thread_id: None, metadata_json, + attachments: vec![], }); } @@ -683,4 +689,34 @@ mod tests { assert_eq!(parsed.channel_id, "123"); assert_eq!(parsed.interaction_id, "456"); } + + #[test] + fn test_parse_slash_command_interaction() { + // Verify that a slash command interaction deserializes correctly. + let json = r#"{ + "type": 2, + "id": "int_1", + "application_id": "app_1", + "channel_id": "ch_1", + "member": { + "user": { + "id": "user_1", + "username": "testuser", + "global_name": "Test User" + } + }, + "data": { + "id": "cmd_1", + "name": "ask", + "options": [ + {"name": "question", "value": "What is rust?"} + ] + }, + "token": "token_abc" + }"#; + + let interaction: DiscordInteraction = serde_json::from_str(json).unwrap(); + assert_eq!(interaction.interaction_type, 2); + assert!(interaction.data.is_some()); + } } diff --git a/channels-src/slack/Cargo.toml b/channels-src/slack/Cargo.toml index 7d77c021..bc8c7434 100644 --- a/channels-src/slack/Cargo.toml +++ b/channels-src/slack/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "slack-channel" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Slack Events API channel for IronClaw" license = "MIT OR Apache-2.0" diff --git a/channels-src/slack/slack.capabilities.json b/channels-src/slack/slack.capabilities.json index 9a16fcd9..7035d925 100644 --- a/channels-src/slack/slack.capabilities.json +++ b/channels-src/slack/slack.capabilities.json @@ -1,6 +1,6 @@ { - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "type": "channel", "name": "slack", "description": "Slack Events API channel for receiving and responding to Slack messages", diff --git a/channels-src/slack/src/lib.rs b/channels-src/slack/src/lib.rs index 75d68e68..71f1e731 100644 --- a/channels-src/slack/src/lib.rs +++ b/channels-src/slack/src/lib.rs @@ -29,7 +29,7 @@ use exports::near::agent::channel::{ AgentResponse, ChannelConfig, Guest, HttpEndpointConfig, IncomingHttpRequest, OutgoingHttpResponse, StatusUpdate, }; -use near::agent::channel_host::{self, EmittedMessage}; +use near::agent::channel_host::{self, EmittedMessage, InboundAttachment}; /// Slack event wrapper. #[derive(Debug, Deserialize)] @@ -78,6 +78,25 @@ struct SlackEvent { /// Subtype (bot_message, etc.) subtype: Option, + + /// File attachments shared in the message. + #[serde(default)] + files: Option>, +} + +/// Slack file attachment. +#[derive(Debug, Deserialize)] +struct SlackFile { + /// File ID. + id: String, + /// MIME type. + mimetype: Option, + /// Original filename. + name: Option, + /// File size in bytes. + size: Option, + /// URL to download the file (requires auth). + url_private: Option, } /// Metadata stored with emitted messages for response routing. @@ -306,13 +325,42 @@ impl Guest for SlackChannel { fn on_status(_update: StatusUpdate) {} + fn on_broadcast(_user_id: String, _response: AgentResponse) -> Result<(), String> { + Err("broadcast not yet implemented for Slack channel".to_string()) + } + fn on_shutdown() { channel_host::log(channel_host::LogLevel::Info, "Slack channel shutting down"); } } +/// Extract attachments from Slack file objects. +fn extract_slack_attachments(files: &Option>) -> Vec { + let Some(files) = files else { + return Vec::new(); + }; + files + .iter() + .map(|f| InboundAttachment { + id: f.id.clone(), + mime_type: f + .mimetype + .clone() + .unwrap_or_else(|| "application/octet-stream".to_string()), + filename: f.name.clone(), + size_bytes: f.size, + source_url: f.url_private.clone(), + storage_key: None, + extracted_text: None, + extras_json: String::new(), + }) + .collect() +} + /// Handle a Slack event and emit message if applicable. fn handle_slack_event(event: SlackEvent, team_id: Option, _event_id: Option) { + let attachments = extract_slack_attachments(&event.files); + match event.event_type.as_str() { // Direct mention of the bot (always in a channel, not a DM) "app_mention" => { @@ -326,7 +374,14 @@ fn handle_slack_event(event: SlackEvent, team_id: Option, _event_id: Opt if !check_sender_permission(&user, &channel, false) { return; } - emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id); + emit_message( + user, + text, + channel, + event.thread_ts.or(Some(ts)), + team_id, + attachments, + ); } } @@ -348,7 +403,14 @@ fn handle_slack_event(event: SlackEvent, team_id: Option, _event_id: Opt if !check_sender_permission(&user, &channel, true) { return; } - emit_message(user, text, channel, event.thread_ts.or(Some(ts)), team_id); + emit_message( + user, + text, + channel, + event.thread_ts.or(Some(ts)), + team_id, + attachments, + ); } } } @@ -369,6 +431,7 @@ fn emit_message( channel: String, thread_ts: Option, team_id: Option, + attachments: Vec, ) { let message_ts = thread_ts.clone().unwrap_or_default(); @@ -396,6 +459,7 @@ fn emit_message( content: cleaned_text, thread_id: thread_ts, metadata_json, + attachments, }); } @@ -551,3 +615,111 @@ fn json_response(status: u16, value: serde_json::Value) -> OutgoingHttpResponse // Export the component export!(SlackChannel); + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_slack_attachments_with_files() { + let files = Some(vec![ + SlackFile { + id: "F123".to_string(), + mimetype: Some("image/png".to_string()), + name: Some("screenshot.png".to_string()), + size: Some(50000), + url_private: Some("https://files.slack.com/F123".to_string()), + }, + SlackFile { + id: "F456".to_string(), + mimetype: Some("application/pdf".to_string()), + name: Some("doc.pdf".to_string()), + size: Some(120000), + url_private: None, + }, + ]); + + let attachments = extract_slack_attachments(&files); + assert_eq!(attachments.len(), 2); + + assert_eq!(attachments[0].id, "F123"); + assert_eq!(attachments[0].mime_type, "image/png"); + assert_eq!(attachments[0].filename, Some("screenshot.png".to_string())); + assert_eq!(attachments[0].size_bytes, Some(50000)); + assert_eq!( + attachments[0].source_url, + Some("https://files.slack.com/F123".to_string()) + ); + + assert_eq!(attachments[1].id, "F456"); + assert_eq!(attachments[1].mime_type, "application/pdf"); + assert!(attachments[1].source_url.is_none()); + } + + #[test] + fn test_extract_slack_attachments_none() { + let attachments = extract_slack_attachments(&None); + assert!(attachments.is_empty()); + } + + #[test] + fn test_extract_slack_attachments_empty() { + let attachments = extract_slack_attachments(&Some(vec![])); + assert!(attachments.is_empty()); + } + + #[test] + fn test_extract_slack_attachments_missing_mime() { + let files = Some(vec![SlackFile { + id: "F789".to_string(), + mimetype: None, + name: Some("unknown".to_string()), + size: None, + url_private: None, + }]); + + let attachments = extract_slack_attachments(&files); + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].mime_type, "application/octet-stream"); + } + + #[test] + fn test_parse_slack_event_with_files() { + let json = r#"{ + "type": "message", + "user": "U123", + "channel": "D456", + "text": "Check this file", + "ts": "1234567890.000001", + "files": [ + { + "id": "F001", + "mimetype": "image/jpeg", + "name": "photo.jpg", + "size": 30000, + "url_private": "https://files.slack.com/F001" + } + ] + }"#; + + let event: SlackEvent = serde_json::from_str(json).unwrap(); + assert!(event.files.is_some()); + let files = event.files.unwrap(); + assert_eq!(files.len(), 1); + assert_eq!(files[0].id, "F001"); + } + + #[test] + fn test_parse_slack_event_without_files() { + let json = r#"{ + "type": "message", + "user": "U123", + "channel": "D456", + "text": "Just text", + "ts": "1234567890.000001" + }"#; + + let event: SlackEvent = serde_json::from_str(json).unwrap(); + assert!(event.files.is_none()); + } +} diff --git a/channels-src/telegram/Cargo.lock b/channels-src/telegram/Cargo.lock index a6e5c3ac..67c27867 100644 --- a/channels-src/telegram/Cargo.lock +++ b/channels-src/telegram/Cargo.lock @@ -212,7 +212,7 @@ dependencies = [ [[package]] name = "telegram-channel" -version = "0.1.0" +version = "0.2.0" dependencies = [ "serde", "serde_json", diff --git a/channels-src/telegram/Cargo.toml b/channels-src/telegram/Cargo.toml index 83e0c8e0..93a1eb57 100644 --- a/channels-src/telegram/Cargo.toml +++ b/channels-src/telegram/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telegram-channel" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Telegram Bot API channel for IronClaw" license = "MIT OR Apache-2.0" diff --git a/channels-src/telegram/src/lib.rs b/channels-src/telegram/src/lib.rs index 6bd33cec..c3ab9050 100644 --- a/channels-src/telegram/src/lib.rs +++ b/channels-src/telegram/src/lib.rs @@ -30,10 +30,10 @@ use serde::{Deserialize, Serialize}; // Re-export generated types use exports::near::agent::channel::{ - AgentResponse, ChannelConfig, Guest, HttpEndpointConfig, IncomingHttpRequest, + AgentResponse, Attachment, ChannelConfig, Guest, HttpEndpointConfig, IncomingHttpRequest, OutgoingHttpResponse, PollConfig, StatusType, StatusUpdate, }; -use near::agent::channel_host::{self, EmittedMessage}; +use near::agent::channel_host::{self, EmittedMessage, InboundAttachment}; // ============================================================================ // Telegram API Types @@ -81,6 +81,87 @@ struct TelegramMessage { /// Bot command entities (for /commands). entities: Option>, + + /// Photo sizes (Telegram sends multiple sizes; last is largest). + #[serde(default)] + photo: Option>, + + /// Document attachment. + document: Option, + + /// Audio attachment. + audio: Option, + + /// Video attachment. + video: Option, + + /// Voice message. + voice: Option, + + /// Sticker. + sticker: Option, +} + +/// Telegram PhotoSize object. +#[derive(Debug, Deserialize)] +struct PhotoSize { + file_id: String, + file_unique_id: String, + width: i32, + height: i32, + file_size: Option, +} + +/// Telegram Document object. +#[derive(Debug, Deserialize)] +struct TelegramDocument { + file_id: String, + file_unique_id: String, + file_name: Option, + mime_type: Option, + file_size: Option, +} + +/// Telegram Audio object. +#[derive(Debug, Deserialize)] +struct TelegramAudio { + file_id: String, + file_unique_id: String, + duration: Option, + file_name: Option, + mime_type: Option, + file_size: Option, +} + +/// Telegram Video object. +#[derive(Debug, Deserialize)] +struct TelegramVideo { + file_id: String, + file_unique_id: String, + duration: Option, + file_name: Option, + mime_type: Option, + file_size: Option, +} + +/// Telegram Voice message object. +#[derive(Debug, Deserialize)] +struct TelegramVoice { + file_id: String, + file_unique_id: String, + duration: u32, + mime_type: Option, + file_size: Option, +} + +/// Telegram Sticker object. +#[derive(Debug, Deserialize)] +struct TelegramSticker { + file_id: String, + file_unique_id: String, + #[serde(rename = "type")] + sticker_type: Option, + file_size: Option, } /// Telegram User object. @@ -139,6 +220,18 @@ struct MessageEntity { user: Option, } +/// Telegram File object returned by getFile. +/// https://core.telegram.org/bots/api#file +#[derive(Debug, Deserialize)] +struct TelegramFile { + /// Identifier for this file. + #[allow(dead_code)] + file_id: String, + + /// File path for downloading. Use https://api.telegram.org/file/bot/. + file_path: Option, +} + /// Telegram API response wrapper. #[derive(Debug, Deserialize)] struct TelegramApiResponse { @@ -236,6 +329,10 @@ struct TelegramConfig { /// Telegram will include this in the X-Telegram-Bot-Api-Secret-Token header. #[serde(default)] webhook_secret: Option, + + /// When true, use polling mode even if tunnel_url is available. + #[serde(default)] + polling_enabled: bool, } // ============================================================================ @@ -363,9 +460,8 @@ impl Guest for TelegramChannel { &config.respond_to_all_group_messages.to_string(), ); - // Mode is determined by whether the host injected a tunnel_url - // If tunnel is configured, use webhooks. Otherwise, use polling. - let webhook_mode = config.tunnel_url.is_some(); + // Mode: use polling if explicitly enabled, otherwise use webhooks when tunnel available. + let webhook_mode = config.tunnel_url.is_some() && !config.polling_enabled; if webhook_mode { channel_host::log( @@ -480,7 +576,7 @@ impl Guest for TelegramChannel { ); let headers_json = serde_json::json!({}).to_string(); - let primary_url = get_updates_url(offset, 30); + let primary_url = get_updates_url(offset, 25); // 35s HTTP timeout outlives Telegram's 30s server-side long-poll. // If the TCP connection drops, retry once immediately with a short poll @@ -584,50 +680,15 @@ impl Guest for TelegramChannel { let metadata: TelegramMessageMetadata = serde_json::from_str(&response.metadata_json) .map_err(|e| format!("Failed to parse metadata: {}", e))?; - // Try sending with Markdown first; fall back to plain text if Telegram - // can't parse the entities (e.g. model leaked with underscores). - let result = send_message( - metadata.chat_id, - &response.content, - Some(metadata.message_id), - Some("Markdown"), - ); + send_response(metadata.chat_id, &response, Some(metadata.message_id)) + } - match result { - Ok(msg_id) => { - channel_host::log( - channel_host::LogLevel::Debug, - &format!( - "Sent message to chat {}: message_id={}", - metadata.chat_id, msg_id - ), - ); - Ok(()) - } - Err(SendError::ParseEntities(detail)) => { - channel_host::log( - channel_host::LogLevel::Warn, - &format!("Markdown parse failed ({}), retrying as plain text", detail), - ); - let msg_id = send_message( - metadata.chat_id, - &response.content, - Some(metadata.message_id), - None, - ) - .map_err(|e| format!("Plain-text retry also failed: {}", e))?; + fn on_broadcast(user_id: String, response: AgentResponse) -> Result<(), String> { + let chat_id: i64 = user_id + .parse() + .map_err(|e| format!("Invalid chat_id '{}': {}", user_id, e))?; - channel_host::log( - channel_host::LogLevel::Debug, - &format!( - "Sent plain-text message to chat {}: message_id={}", - metadata.chat_id, msg_id - ), - ); - Ok(()) - } - Err(e) => Err(e.to_string()), - } + send_response(chat_id, &response, None) } fn on_status(update: StatusUpdate) { @@ -813,6 +874,324 @@ fn send_message( } } +// ============================================================================ +// Voice File Download +// ============================================================================ + +/// Download a voice file from Telegram by file_id. +/// +/// 1. Call getFile to get the file_path. +/// 2. Download the file bytes from /file/bot{TOKEN}/{file_path}. +/// Percent-encode a string for safe use as a URL query parameter value. +fn percent_encode(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + for b in s.bytes() { + match b { + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'-' | b'_' | b'.' | b'~' => { + out.push(b as char); + } + _ => { + out.push_str(&format!("%{:02X}", b)); + } + } + } + out +} + +fn download_telegram_file(file_id: &str) -> Result, String> { + // Reject file_id containing curly braces to prevent credential placeholder injection + if file_id.contains('{') || file_id.contains('}') { + return Err("invalid file_id: contains forbidden characters".to_string()); + } + + // Step 1: Call getFile to get file_path + let get_file_url = format!( + "https://api.telegram.org/bot{{TELEGRAM_BOT_TOKEN}}/getFile?file_id={}", + percent_encode(file_id) + ); + + let headers = serde_json::json!({}); + let result = + channel_host::http_request("GET", &get_file_url, &headers.to_string(), None, None); + + let response = result.map_err(|e| format!("getFile request failed: {}", e))?; + + if response.status != 200 { + let body_str = String::from_utf8_lossy(&response.body); + return Err(format!("getFile returned {}: {}", response.status, body_str)); + } + + let api_response: TelegramApiResponse = + serde_json::from_slice(&response.body) + .map_err(|e| format!("Failed to parse getFile response: {}", e))?; + + if !api_response.ok { + return Err(format!( + "getFile API error: {}", + api_response + .description + .unwrap_or_else(|| "unknown".to_string()) + )); + } + + let file = api_response + .result + .ok_or_else(|| "getFile returned no result".to_string())?; + + let file_path = file + .file_path + .ok_or_else(|| "getFile returned no file_path".to_string())?; + + // Sanitize file_path against credential placeholder injection + if file_path.contains('{') || file_path.contains('}') { + return Err("invalid file_path: contains forbidden characters".to_string()); + } + + // Step 2: Download the actual file bytes + let download_url = format!( + "https://api.telegram.org/file/bot{{TELEGRAM_BOT_TOKEN}}/{}", + file_path + ); + + let result = + channel_host::http_request("GET", &download_url, &headers.to_string(), None, None); + + let response = result.map_err(|e| format!("File download failed: {}", e))?; + + if response.status != 200 { + return Err(format!( + "File download returned status {}", + response.status + )); + } + + Ok(response.body) +} + +// ============================================================================ +// Attachment Sending (Photo / Document) +// ============================================================================ + +/// Maximum photo size for Telegram sendPhoto (10 MB). +const MAX_PHOTO_SIZE: usize = 10 * 1024 * 1024; + +/// Write a multipart/form-data text field. +fn write_multipart_field(body: &mut Vec, boundary: &str, name: &str, value: &str) { + body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes()); + body.extend_from_slice( + format!("Content-Disposition: form-data; name=\"{}\"\r\n\r\n", name).as_bytes(), + ); + body.extend_from_slice(value.as_bytes()); + body.extend_from_slice(b"\r\n"); +} + +/// Write a multipart/form-data file field. +fn write_multipart_file( + body: &mut Vec, + boundary: &str, + field: &str, + filename: &str, + content_type: &str, + data: &[u8], +) { + // Sanitize filename: strip quotes, newlines, and non-ASCII to prevent header injection + let safe_filename: String = filename + .chars() + .filter(|c| *c != '"' && *c != '\r' && *c != '\n' && *c != '\\' && c.is_ascii()) + .collect(); + let safe_filename = if safe_filename.is_empty() { + "file".to_string() + } else { + safe_filename + }; + body.extend_from_slice(format!("--{}\r\n", boundary).as_bytes()); + body.extend_from_slice( + format!( + "Content-Disposition: form-data; name=\"{}\"; filename=\"{}\"\r\n", + field, safe_filename + ) + .as_bytes(), + ); + body.extend_from_slice(format!("Content-Type: {}\r\n\r\n", content_type).as_bytes()); + body.extend_from_slice(data); + body.extend_from_slice(b"\r\n"); +} + +/// Send a photo via the Telegram Bot API (multipart upload). +/// +/// Falls back to `send_document()` if the photo exceeds 10 MB. +fn send_photo( + chat_id: i64, + filename: &str, + mime_type: &str, + data: &[u8], + reply_to_message_id: Option, +) -> Result<(), String> { + if data.len() > MAX_PHOTO_SIZE { + channel_host::log( + channel_host::LogLevel::Info, + &format!( + "Photo {} exceeds 10MB ({}), sending as document", + filename, + data.len() + ), + ); + return send_document(chat_id, filename, mime_type, data, reply_to_message_id); + } + + let boundary = format!("ironclaw-{}", channel_host::now_millis()); + let mut body = Vec::new(); + + write_multipart_field(&mut body, &boundary, "chat_id", &chat_id.to_string()); + if let Some(msg_id) = reply_to_message_id { + write_multipart_field(&mut body, &boundary, "reply_to_message_id", &msg_id.to_string()); + } + write_multipart_file(&mut body, &boundary, "photo", filename, mime_type, data); + body.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes()); + + let headers = serde_json::json!({ + "Content-Type": format!("multipart/form-data; boundary={}", boundary) + }); + + let result = channel_host::http_request( + "POST", + "https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendPhoto", + &headers.to_string(), + Some(&body), + Some(60_000), // 60s timeout for file uploads + ); + + match result { + Ok(resp) if resp.status == 200 => { + channel_host::log( + channel_host::LogLevel::Debug, + &format!("Sent photo '{}' to chat {}", filename, chat_id), + ); + Ok(()) + } + Ok(resp) => { + let body_str = String::from_utf8_lossy(&resp.body); + Err(format!( + "sendPhoto failed (HTTP {}): {}", + resp.status, body_str + )) + } + Err(e) => Err(format!("sendPhoto HTTP request failed: {}", e)), + } +} + +/// Send a document via the Telegram Bot API (multipart upload). +fn send_document( + chat_id: i64, + filename: &str, + mime_type: &str, + data: &[u8], + reply_to_message_id: Option, +) -> Result<(), String> { + let boundary = format!("ironclaw-{}", channel_host::now_millis()); + let mut body = Vec::new(); + + write_multipart_field(&mut body, &boundary, "chat_id", &chat_id.to_string()); + if let Some(msg_id) = reply_to_message_id { + write_multipart_field(&mut body, &boundary, "reply_to_message_id", &msg_id.to_string()); + } + write_multipart_file(&mut body, &boundary, "document", filename, mime_type, data); + body.extend_from_slice(format!("--{}--\r\n", boundary).as_bytes()); + + let headers = serde_json::json!({ + "Content-Type": format!("multipart/form-data; boundary={}", boundary) + }); + + let result = channel_host::http_request( + "POST", + "https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendDocument", + &headers.to_string(), + Some(&body), + Some(60_000), // 60s timeout for file uploads + ); + + match result { + Ok(resp) if resp.status == 200 => { + channel_host::log( + channel_host::LogLevel::Debug, + &format!("Sent document '{}' to chat {}", filename, chat_id), + ); + Ok(()) + } + Ok(resp) => { + let body_str = String::from_utf8_lossy(&resp.body); + Err(format!( + "sendDocument failed (HTTP {}): {}", + resp.status, body_str + )) + } + Err(e) => Err(format!("sendDocument HTTP request failed: {}", e)), + } +} + +/// Image MIME types that Telegram's sendPhoto API supports. +const PHOTO_MIME_TYPES: &[&str] = &[ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", +]; + +/// Send a full agent response (attachments + text) to a chat. +/// +/// Shared implementation for both `on_respond` and `on_broadcast`. +fn send_response( + chat_id: i64, + response: &AgentResponse, + reply_to_message_id: Option, +) -> Result<(), String> { + // Send attachments first (photos/documents) + for attachment in &response.attachments { + send_attachment(chat_id, attachment, reply_to_message_id)?; + } + + // Skip text if empty and we already sent attachments + if response.content.is_empty() && !response.attachments.is_empty() { + return Ok(()); + } + + // Try Markdown, fall back to plain text on parse errors + match send_message(chat_id, &response.content, reply_to_message_id, Some("Markdown")) { + Ok(_) => Ok(()), + Err(SendError::ParseEntities(_)) => { + send_message(chat_id, &response.content, reply_to_message_id, None) + .map(|_| ()) + .map_err(|e| format!("Plain-text retry also failed: {}", e)) + } + Err(e) => Err(e.to_string()), + } +} + +/// Send a single attachment, choosing sendPhoto or sendDocument based on MIME type. +fn send_attachment( + chat_id: i64, + attachment: &Attachment, + reply_to_message_id: Option, +) -> Result<(), String> { + if PHOTO_MIME_TYPES.contains(&attachment.mime_type.as_str()) { + send_photo( + chat_id, + &attachment.filename, + &attachment.mime_type, + &attachment.data, + reply_to_message_id, + ) + } else { + send_document( + chat_id, + &attachment.filename, + &attachment.mime_type, + &attachment.data, + reply_to_message_id, + ) + } +} + // ============================================================================ // Webhook Management // ============================================================================ @@ -990,16 +1369,264 @@ fn handle_update(update: TelegramUpdate) { } } +/// Build extras-json with optional duration. +fn extras_json(duration_secs: Option) -> String { + match duration_secs { + Some(d) => format!(r#"{{"duration_secs":{}}}"#, d), + None => String::new(), + } +} + +/// Build an inbound attachment with the standard fields. +fn make_inbound_attachment( + id: String, + mime_type: String, + filename: Option, + size_bytes: Option, + source_url: Option, + extracted_text: Option, + duration_secs: Option, +) -> InboundAttachment { + InboundAttachment { + id, + mime_type, + filename, + size_bytes, + source_url, + storage_key: None, + extracted_text, + extras_json: extras_json(duration_secs), + } +} + +/// Extract attachments from a Telegram message. +fn extract_attachments(message: &TelegramMessage) -> Vec { + let mut attachments = Vec::new(); + let get_file_url = |file_id: &str| { + format!( + "https://api.telegram.org/bot{{TELEGRAM_BOT_TOKEN}}/getFile?file_id={}", + percent_encode(file_id) + ) + }; + + // Photo: Telegram sends multiple sizes; use the largest (last). + if let Some(ref photos) = message.photo { + if let Some(largest) = photos.last() { + attachments.push(make_inbound_attachment( + largest.file_id.clone(), + "image/jpeg".to_string(), + None, + largest.file_size.map(|s| s as u64), + Some(get_file_url(&largest.file_id)), + None, + None, + )); + } + } + + // Document + if let Some(ref doc) = message.document { + attachments.push(make_inbound_attachment( + doc.file_id.clone(), + doc.mime_type.clone().unwrap_or_else(|| "application/octet-stream".to_string()), + doc.file_name.clone(), + doc.file_size.map(|s| s as u64), + Some(get_file_url(&doc.file_id)), + None, + None, + )); + } + + // Audio + if let Some(ref audio) = message.audio { + attachments.push(make_inbound_attachment( + audio.file_id.clone(), + audio.mime_type.clone().unwrap_or_else(|| "audio/mpeg".to_string()), + audio.file_name.clone(), + audio.file_size.map(|s| s as u64), + Some(get_file_url(&audio.file_id)), + None, + audio.duration, + )); + } + + // Video + if let Some(ref video) = message.video { + attachments.push(make_inbound_attachment( + video.file_id.clone(), + video.mime_type.clone().unwrap_or_else(|| "video/mp4".to_string()), + video.file_name.clone(), + video.file_size.map(|s| s as u64), + Some(get_file_url(&video.file_id)), + None, + video.duration, + )); + } + + // Voice + if let Some(ref voice) = message.voice { + let mime_type = voice + .mime_type + .clone() + .unwrap_or_else(|| "audio/ogg".to_string()); + + attachments.push(make_inbound_attachment( + voice.file_id.clone(), + mime_type, + Some(format!("voice_{}.ogg", voice.file_id)), + voice.file_size.map(|s| s as u64), + Some(get_file_url(&voice.file_id)), + None, + Some(voice.duration), + )); + } + + // Sticker + if let Some(ref sticker) = message.sticker { + attachments.push(make_inbound_attachment( + sticker.file_id.clone(), + "image/webp".to_string(), + None, + sticker.file_size.map(|s| s as u64), + Some(get_file_url(&sticker.file_id)), + None, + None, + )); + } + + attachments +} + +/// Download voice file bytes and store them via the host for transcription. +/// +/// Separated from `extract_attachments` so that function stays pure (no host +/// calls) and remains testable in native unit tests. +fn download_and_store_voice(attachments: &[InboundAttachment]) { + for att in attachments { + // Voice attachments have a generated filename like "voice_.ogg" + let is_voice = att + .filename + .as_ref() + .is_some_and(|f| f.starts_with("voice_")); + if !is_voice { + continue; + } + + match download_telegram_file(&att.id) { + Ok(bytes) => { + channel_host::log( + channel_host::LogLevel::Info, + &format!("Downloaded voice file: {} bytes", bytes.len()), + ); + if let Err(e) = channel_host::store_attachment_data(&att.id, &bytes) { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Failed to store voice data: {}", e), + ); + } + } + Err(e) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Failed to download voice file: {}", e), + ); + } + } + } +} + +/// Returns true if the attachment should be downloaded for document text extraction. +/// +/// Excludes voice (handled by transcription), image (vision pipeline), +/// audio (transcription), and video attachments. +fn is_downloadable_document(att: &InboundAttachment) -> bool { + let is_voice = att + .filename + .as_ref() + .is_some_and(|f| f.starts_with("voice_")); + if is_voice { + return false; + } + if att.mime_type.starts_with("image/") + || att.mime_type.starts_with("audio/") + || att.mime_type.starts_with("video/") + { + return false; + } + true +} + +/// Download document file bytes and store them via the host for text extraction. +/// +/// Downloads any attachment that isn't voice or image so the host-side +/// `DocumentExtractionMiddleware` can extract text from PDFs, Office docs, etc. +/// +/// On failure, sets `extracted_text` to an error message so the user gets feedback. +fn download_and_store_documents(attachments: &mut [InboundAttachment]) { + for att in attachments.iter_mut() { + if !is_downloadable_document(att) { + continue; + } + + match download_telegram_file(&att.id) { + Ok(bytes) => { + channel_host::log( + channel_host::LogLevel::Info, + &format!( + "Downloaded document file: {} bytes, mime={}", + bytes.len(), + att.mime_type + ), + ); + if let Err(e) = channel_host::store_attachment_data(&att.id, &bytes) { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Failed to store document data: {}", e), + ); + } + } + Err(e) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Failed to download document file: {}", e), + ); + let name = att.filename.as_deref().unwrap_or("document"); + att.extracted_text = Some(format!( + "[Failed to download '{name}': {e}. \ + The file may be too large or unavailable. Please try a smaller file.]" + )); + } + } + } +} + /// Process a single message. fn handle_message(message: TelegramMessage) { + // Extract attachments from media fields (pure data mapping, no host calls) + let mut attachments = extract_attachments(&message); + + // Download and store voice attachments for host-side transcription + download_and_store_voice(&attachments); + + // Download and store document attachments for host-side text extraction + download_and_store_documents(&mut attachments); + // Use text or caption (for media messages) + let has_voice = message.voice.is_some(); let content = message .text .filter(|t| !t.is_empty()) .or_else(|| message.caption.filter(|c| !c.is_empty())) - .unwrap_or_default(); + .unwrap_or_else(|| { + if has_voice { + "[Voice note]".to_string() + } else { + String::new() + } + }); - if content.is_empty() { + // Allow messages with attachments even if text content is empty + if content.is_empty() && attachments.is_empty() { return; } @@ -1155,6 +1782,8 @@ fn handle_message(message: TelegramMessage) { }, ) { Some(value) => value, + // Allow attachment-only messages even without text + None if !attachments.is_empty() => String::new(), None => return, }; @@ -1165,6 +1794,7 @@ fn handle_message(message: TelegramMessage) { content: content_to_emit, thread_id: None, // Telegram doesn't have threads in the same way metadata_json, + attachments, }); channel_host::log( @@ -1740,4 +2370,239 @@ mod tests { assert!(msg.len() <= TELEGRAM_STATUS_MAX_CHARS + 3); assert!(msg.ends_with("...")); } + + // === Attachment extraction fixture tests === + + #[test] + fn test_extract_attachments_photo() { + let json = r#"{ + "message_id": 1, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "caption": "What is this?", + "photo": [ + {"file_id": "small_id", "file_unique_id": "s1", "width": 90, "height": 90, "file_size": 1234}, + {"file_id": "large_id", "file_unique_id": "l1", "width": 800, "height": 600, "file_size": 54321} + ] + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "large_id"); // Largest photo + assert_eq!(attachments[0].mime_type, "image/jpeg"); + assert_eq!(attachments[0].size_bytes, Some(54321)); + assert!(attachments[0].source_url.as_ref().unwrap().contains("large_id")); + } + + #[test] + fn test_extract_attachments_document() { + let json = r#"{ + "message_id": 2, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "document": { + "file_id": "doc_abc", + "file_unique_id": "d1", + "file_name": "report.pdf", + "mime_type": "application/pdf", + "file_size": 102400 + }, + "caption": "Here is the report" + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "doc_abc"); + assert_eq!(attachments[0].mime_type, "application/pdf"); + assert_eq!(attachments[0].filename, Some("report.pdf".to_string())); + assert_eq!(attachments[0].size_bytes, Some(102400)); + } + + #[test] + fn test_extract_attachments_voice() { + let json = r#"{ + "message_id": 3, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "voice": { + "file_id": "voice_xyz", + "file_unique_id": "v1", + "duration": 5, + "mime_type": "audio/ogg", + "file_size": 9000 + } + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "voice_xyz"); + assert_eq!(attachments[0].mime_type, "audio/ogg"); + assert_eq!( + attachments[0].filename.as_deref(), + Some("voice_voice_xyz.ogg") + ); + assert!(attachments[0] + .extras_json + .contains("\"duration_secs\":5")); + } + + #[test] + fn test_extract_attachments_video() { + let json = r#"{ + "message_id": 4, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "video": { + "file_id": "vid_1", + "file_unique_id": "vv1", + "file_name": "clip.mp4", + "mime_type": "video/mp4", + "file_size": 5000000 + }, + "caption": "Check this out" + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "vid_1"); + assert_eq!(attachments[0].mime_type, "video/mp4"); + assert_eq!(attachments[0].filename, Some("clip.mp4".to_string())); + } + + #[test] + fn test_extract_attachments_audio() { + let json = r#"{ + "message_id": 5, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "audio": { + "file_id": "audio_1", + "file_unique_id": "a1", + "file_name": "song.mp3", + "mime_type": "audio/mpeg", + "file_size": 3000000 + } + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "audio_1"); + assert_eq!(attachments[0].mime_type, "audio/mpeg"); + assert_eq!(attachments[0].filename, Some("song.mp3".to_string())); + } + + #[test] + fn test_extract_attachments_sticker() { + let json = r#"{ + "message_id": 6, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "sticker": { + "file_id": "sticker_1", + "file_unique_id": "st1", + "type": "regular", + "file_size": 20000 + } + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "sticker_1"); + assert_eq!(attachments[0].mime_type, "image/webp"); + } + + #[test] + fn test_extract_attachments_text_only_empty() { + let json = r#"{ + "message_id": 7, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "text": "Hello" + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + assert!(attachments.is_empty()); + } + + #[test] + fn test_extract_attachments_multiple_types() { + let json = r#"{ + "message_id": 8, + "from": {"id": 1, "is_bot": false, "first_name": "A"}, + "chat": {"id": 1, "type": "private"}, + "photo": [ + {"file_id": "photo_1", "file_unique_id": "p1", "width": 100, "height": 100} + ], + "document": { + "file_id": "doc_1", + "file_unique_id": "d1", + "file_name": "file.txt", + "mime_type": "text/plain" + } + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + let attachments = extract_attachments(&msg); + + // Both photo and document should be extracted + assert_eq!(attachments.len(), 2); + } + + #[test] + fn test_parse_update_with_photo_fallback_content() { + // A photo-only message (no text, no caption) should have empty content + // but still produce attachments + let json = r#"{ + "message_id": 9, + "from": {"id": 42, "is_bot": false, "first_name": "Test"}, + "chat": {"id": 42, "type": "private"}, + "photo": [ + {"file_id": "ph1", "file_unique_id": "u1", "width": 320, "height": 240} + ] + }"#; + let msg: TelegramMessage = serde_json::from_str(json).unwrap(); + + // Content is empty (no text, no caption) + assert!(msg.text.is_none()); + assert!(msg.caption.is_none()); + + // But attachments exist + let attachments = extract_attachments(&msg); + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "ph1"); + } + + #[test] + fn test_is_downloadable_document() { + let make = |mime: &str, filename: Option<&str>| InboundAttachment { + id: "test".to_string(), + mime_type: mime.to_string(), + filename: filename.map(|s| s.to_string()), + size_bytes: Some(1024), + source_url: None, + storage_key: None, + extracted_text: None, + extras_json: String::new(), + }; + + // PDFs and Office docs should be downloaded + assert!(is_downloadable_document(&make("application/pdf", Some("report.pdf")))); + assert!(is_downloadable_document(&make( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + Some("doc.docx"), + ))); + assert!(is_downloadable_document(&make("text/plain", Some("notes.txt")))); + + // Voice, image, audio, video should NOT be downloaded + assert!(!is_downloadable_document(&make("audio/ogg", Some("voice_123.ogg")))); + assert!(!is_downloadable_document(&make("image/jpeg", None))); + assert!(!is_downloadable_document(&make("audio/mpeg", Some("song.mp3")))); + assert!(!is_downloadable_document(&make("video/mp4", Some("clip.mp4")))); + } } diff --git a/channels-src/telegram/telegram.capabilities.json b/channels-src/telegram/telegram.capabilities.json index c6a08f27..8317307b 100644 --- a/channels-src/telegram/telegram.capabilities.json +++ b/channels-src/telegram/telegram.capabilities.json @@ -1,6 +1,6 @@ { - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "type": "channel", "name": "telegram", "description": "Telegram Bot API channel for receiving and responding to Telegram messages", @@ -17,7 +17,8 @@ "capabilities": { "http": { "allowlist": [ - { "host": "api.telegram.org", "path_prefix": "/bot" } + { "host": "api.telegram.org", "path_prefix": "/bot" }, + { "host": "api.telegram.org", "path_prefix": "/file/bot" } ], "credentials": { "telegram_bot": { @@ -26,6 +27,7 @@ "host_patterns": ["api.telegram.org"] } }, + "max_response_bytes": 52428800, "rate_limit": { "requests_per_minute": 30, "requests_per_hour": 1000 diff --git a/channels-src/whatsapp/Cargo.toml b/channels-src/whatsapp/Cargo.toml index 4e334bee..cf211e2e 100644 --- a/channels-src/whatsapp/Cargo.toml +++ b/channels-src/whatsapp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "whatsapp-channel" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "WhatsApp Cloud API channel for IronClaw" diff --git a/channels-src/whatsapp/src/lib.rs b/channels-src/whatsapp/src/lib.rs index c60fea55..c69a9b9f 100644 --- a/channels-src/whatsapp/src/lib.rs +++ b/channels-src/whatsapp/src/lib.rs @@ -32,7 +32,7 @@ use exports::near::agent::channel::{ AgentResponse, ChannelConfig, Guest, HttpEndpointConfig, IncomingHttpRequest, OutgoingHttpResponse, StatusUpdate, }; -use near::agent::channel_host::{self, EmittedMessage}; +use near::agent::channel_host::{self, EmittedMessage, InboundAttachment}; // ============================================================================ // WhatsApp Cloud API Types @@ -137,10 +137,46 @@ struct WhatsAppMessage { /// Text content (if type is "text") text: Option, + /// Image content + image: Option, + + /// Audio content + audio: Option, + + /// Video content + video: Option, + + /// Document content + document: Option, + /// Context for replies context: Option, } +/// WhatsApp media attachment (image, audio, video). +#[derive(Debug, Deserialize)] +struct WhatsAppMedia { + /// Media ID (use to download via Graph API) + id: String, + /// MIME type + mime_type: Option, + /// Caption text + caption: Option, +} + +/// WhatsApp document attachment. +#[derive(Debug, Deserialize)] +struct WhatsAppDocument { + /// Media ID + id: String, + /// MIME type + mime_type: Option, + /// Filename + filename: Option, + /// Caption text + caption: Option, +} + /// Text message content. #[derive(Debug, Deserialize)] struct TextContent { @@ -476,6 +512,10 @@ impl Guest for WhatsAppChannel { fn on_status(_update: StatusUpdate) {} + fn on_broadcast(_user_id: String, _response: AgentResponse) -> Result<(), String> { + Err("broadcast not yet implemented for WhatsApp channel".to_string()) + } + fn on_shutdown() { channel_host::log( channel_host::LogLevel::Info, @@ -618,26 +658,102 @@ fn handle_incoming_message(req: &IncomingHttpRequest) -> OutgoingHttpResponse { json_response(200, serde_json::json!({"status": "ok"})) } +/// Extract attachments from a WhatsApp message. +fn extract_whatsapp_attachments(message: &WhatsAppMessage) -> Vec { + let mut attachments = Vec::new(); + + if let Some(ref img) = message.image { + attachments.push(InboundAttachment { + id: img.id.clone(), + mime_type: img + .mime_type + .clone() + .unwrap_or_else(|| "image/jpeg".to_string()), + filename: None, + size_bytes: None, + source_url: None, // WhatsApp requires Graph API call with media ID to get URL + storage_key: None, + extracted_text: img.caption.clone(), + extras_json: String::new(), + }); + } + + if let Some(ref audio) = message.audio { + attachments.push(InboundAttachment { + id: audio.id.clone(), + mime_type: audio + .mime_type + .clone() + .unwrap_or_else(|| "audio/ogg".to_string()), + filename: None, + size_bytes: None, + source_url: None, + storage_key: None, + extracted_text: audio.caption.clone(), + extras_json: String::new(), + }); + } + + if let Some(ref video) = message.video { + attachments.push(InboundAttachment { + id: video.id.clone(), + mime_type: video + .mime_type + .clone() + .unwrap_or_else(|| "video/mp4".to_string()), + filename: None, + size_bytes: None, + source_url: None, + storage_key: None, + extracted_text: video.caption.clone(), + extras_json: String::new(), + }); + } + + if let Some(ref doc) = message.document { + attachments.push(InboundAttachment { + id: doc.id.clone(), + mime_type: doc + .mime_type + .clone() + .unwrap_or_else(|| "application/octet-stream".to_string()), + filename: doc.filename.clone(), + size_bytes: None, + source_url: None, + storage_key: None, + extracted_text: doc.caption.clone(), + extras_json: String::new(), + }); + } + + attachments +} + /// Process a single WhatsApp message. fn handle_message( message: &WhatsAppMessage, phone_number_id: &str, contact_names: &std::collections::HashMap, ) { - // Only handle text messages for now - // TODO: Add support for image, audio, video, document, etc. - if message.message_type != "text" { - channel_host::log( - channel_host::LogLevel::Debug, - &format!("Skipping non-text message type: {}", message.message_type), - ); - return; - } + let attachments = extract_whatsapp_attachments(message); - // Extract text content + // Extract text content (from text body or media captions) let text = match &message.text { Some(t) if !t.body.is_empty() => t.body.clone(), - _ => return, + _ => { + // Try to use caption from media messages as content + let caption = message + .image + .as_ref() + .and_then(|m| m.caption.clone()) + .or_else(|| message.video.as_ref().and_then(|m| m.caption.clone())) + .or_else(|| message.document.as_ref().and_then(|m| m.caption.clone())); + match caption { + Some(c) if !c.is_empty() => c, + _ if !attachments.is_empty() => String::new(), + _ => return, + } + } }; // Look up sender's name from contacts @@ -670,6 +786,7 @@ fn handle_message( content: text, thread_id: None, // WhatsApp doesn't have threads like Slack/Discord metadata_json, + attachments, }); channel_host::log( @@ -947,4 +1064,138 @@ mod tests { assert_eq!(parsed.phone_number_id, "123456"); assert_eq!(parsed.sender_phone, "15551234567"); } + + // === Attachment extraction fixture tests === + + #[test] + fn test_extract_whatsapp_image_attachment() { + let msg = WhatsAppMessage { + id: "msg1".to_string(), + from: "15551234567".to_string(), + timestamp: "1234567890".to_string(), + message_type: "image".to_string(), + text: None, + image: Some(WhatsAppMedia { + id: "media_img_1".to_string(), + mime_type: Some("image/jpeg".to_string()), + caption: Some("Look at this".to_string()), + }), + audio: None, + video: None, + document: None, + context: None, + }; + + let attachments = extract_whatsapp_attachments(&msg); + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "media_img_1"); + assert_eq!(attachments[0].mime_type, "image/jpeg"); + assert_eq!( + attachments[0].extracted_text, + Some("Look at this".to_string()) + ); + } + + #[test] + fn test_extract_whatsapp_document_attachment() { + let msg = WhatsAppMessage { + id: "msg2".to_string(), + from: "15551234567".to_string(), + timestamp: "1234567890".to_string(), + message_type: "document".to_string(), + text: None, + image: None, + audio: None, + video: None, + document: Some(WhatsAppDocument { + id: "media_doc_1".to_string(), + mime_type: Some("application/pdf".to_string()), + filename: Some("report.pdf".to_string()), + caption: None, + }), + context: None, + }; + + let attachments = extract_whatsapp_attachments(&msg); + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "media_doc_1"); + assert_eq!(attachments[0].mime_type, "application/pdf"); + assert_eq!( + attachments[0].filename, + Some("report.pdf".to_string()) + ); + } + + #[test] + fn test_extract_whatsapp_audio_video_attachments() { + let msg = WhatsAppMessage { + id: "msg3".to_string(), + from: "15551234567".to_string(), + timestamp: "1234567890".to_string(), + message_type: "audio".to_string(), + text: None, + image: None, + audio: Some(WhatsAppMedia { + id: "media_audio_1".to_string(), + mime_type: Some("audio/ogg".to_string()), + caption: None, + }), + video: Some(WhatsAppMedia { + id: "media_video_1".to_string(), + mime_type: Some("video/mp4".to_string()), + caption: None, + }), + document: None, + context: None, + }; + + let attachments = extract_whatsapp_attachments(&msg); + assert_eq!(attachments.len(), 2); + assert_eq!(attachments[0].id, "media_audio_1"); + assert_eq!(attachments[1].id, "media_video_1"); + } + + #[test] + fn test_extract_whatsapp_text_only_no_attachments() { + let msg = WhatsAppMessage { + id: "msg4".to_string(), + from: "15551234567".to_string(), + timestamp: "1234567890".to_string(), + message_type: "text".to_string(), + text: Some(TextContent { + body: "Hello".to_string(), + }), + image: None, + audio: None, + video: None, + document: None, + context: None, + }; + + let attachments = extract_whatsapp_attachments(&msg); + assert!(attachments.is_empty()); + } + + #[test] + fn test_parse_whatsapp_image_message() { + let json = r#"{ + "id": "wamid.123", + "from": "15551234567", + "timestamp": "1234567890", + "type": "image", + "image": { + "id": "media_img_abc", + "mime_type": "image/jpeg", + "caption": "Check this" + } + }"#; + + let msg: WhatsAppMessage = serde_json::from_str(json).unwrap(); + assert_eq!(msg.message_type, "image"); + assert!(msg.image.is_some()); + + let attachments = extract_whatsapp_attachments(&msg); + assert_eq!(attachments.len(), 1); + assert_eq!(attachments[0].id, "media_img_abc"); + } } diff --git a/channels-src/whatsapp/whatsapp.capabilities.json b/channels-src/whatsapp/whatsapp.capabilities.json index 78786305..a0115d79 100644 --- a/channels-src/whatsapp/whatsapp.capabilities.json +++ b/channels-src/whatsapp/whatsapp.capabilities.json @@ -1,6 +1,6 @@ { - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "type": "channel", "name": "whatsapp", "description": "WhatsApp Cloud API channel for receiving and responding to WhatsApp messages", diff --git a/registry/channels/discord.json b/registry/channels/discord.json index 1ffd0e30..1b13658a 100644 --- a/registry/channels/discord.json +++ b/registry/channels/discord.json @@ -2,8 +2,8 @@ "name": "discord", "display_name": "Discord Channel", "kind": "channel", - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "description": "Talk to your agent in Discord", "keywords": [ "messaging", diff --git a/registry/channels/slack.json b/registry/channels/slack.json index f1e68a43..bd1e60ed 100644 --- a/registry/channels/slack.json +++ b/registry/channels/slack.json @@ -2,8 +2,8 @@ "name": "slack", "display_name": "Slack Channel", "kind": "channel", - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "description": "Talk to your agent in Slack", "keywords": [ "messaging", diff --git a/registry/channels/telegram.json b/registry/channels/telegram.json index b8354834..01405b2c 100644 --- a/registry/channels/telegram.json +++ b/registry/channels/telegram.json @@ -2,8 +2,8 @@ "name": "telegram", "display_name": "Telegram Channel", "kind": "channel", - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "description": "Talk to your agent through a Telegram bot", "keywords": [ "messaging", diff --git a/registry/channels/whatsapp.json b/registry/channels/whatsapp.json index 6c4e7f65..5e7c2bc3 100644 --- a/registry/channels/whatsapp.json +++ b/registry/channels/whatsapp.json @@ -2,8 +2,8 @@ "name": "whatsapp", "display_name": "WhatsApp Channel", "kind": "channel", - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "description": "Talk to your agent through WhatsApp", "keywords": [ "messaging", diff --git a/registry/tools/github.json b/registry/tools/github.json index 273a29f1..bf7af291 100644 --- a/registry/tools/github.json +++ b/registry/tools/github.json @@ -2,8 +2,8 @@ "name": "github", "display_name": "GitHub", "kind": "tool", - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "description": "GitHub integration for issues, PRs, repos, and code search", "keywords": [ "git", diff --git a/registry/tools/gmail.json b/registry/tools/gmail.json index b8d10945..2bdf6350 100644 --- a/registry/tools/gmail.json +++ b/registry/tools/gmail.json @@ -2,8 +2,8 @@ "name": "gmail", "display_name": "Gmail", "kind": "tool", - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "description": "Read, send, and manage Gmail messages and threads", "keywords": [ "email", diff --git a/registry/tools/google-calendar.json b/registry/tools/google-calendar.json index 376c73fb..7b0afd80 100644 --- a/registry/tools/google-calendar.json +++ b/registry/tools/google-calendar.json @@ -2,8 +2,8 @@ "name": "google-calendar", "display_name": "Google Calendar", "kind": "tool", - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "description": "Create, read, update, and delete Google Calendar events", "keywords": [ "calendar", diff --git a/registry/tools/google-docs.json b/registry/tools/google-docs.json index 5f5545d4..b564d0e6 100644 --- a/registry/tools/google-docs.json +++ b/registry/tools/google-docs.json @@ -2,8 +2,8 @@ "name": "google-docs", "display_name": "Google Docs", "kind": "tool", - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "description": "Create and edit Google Docs documents", "keywords": [ "documents", diff --git a/registry/tools/google-drive.json b/registry/tools/google-drive.json index d2c540e9..180aaa1e 100644 --- a/registry/tools/google-drive.json +++ b/registry/tools/google-drive.json @@ -2,8 +2,8 @@ "name": "google-drive", "display_name": "Google Drive", "kind": "tool", - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "description": "Upload, download, search, and manage Google Drive files and folders", "keywords": [ "storage", diff --git a/registry/tools/google-sheets.json b/registry/tools/google-sheets.json index f82f8778..82575182 100644 --- a/registry/tools/google-sheets.json +++ b/registry/tools/google-sheets.json @@ -2,8 +2,8 @@ "name": "google-sheets", "display_name": "Google Sheets", "kind": "tool", - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "description": "Read and write Google Sheets spreadsheet data", "keywords": [ "spreadsheets", diff --git a/registry/tools/google-slides.json b/registry/tools/google-slides.json index e0373acf..5127b17d 100644 --- a/registry/tools/google-slides.json +++ b/registry/tools/google-slides.json @@ -2,8 +2,8 @@ "name": "google-slides", "display_name": "Google Slides", "kind": "tool", - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "description": "Create and edit Google Slides presentations", "keywords": [ "presentations", diff --git a/registry/tools/slack.json b/registry/tools/slack.json index c19361cd..fe038438 100644 --- a/registry/tools/slack.json +++ b/registry/tools/slack.json @@ -2,8 +2,8 @@ "name": "slack-tool", "display_name": "Slack Tool", "kind": "tool", - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "description": "Your agent uses Slack to post and read messages in your workspace", "keywords": [ "messaging", diff --git a/registry/tools/telegram.json b/registry/tools/telegram.json index a2a24ae7..ab036396 100644 --- a/registry/tools/telegram.json +++ b/registry/tools/telegram.json @@ -2,8 +2,8 @@ "name": "telegram-mtproto", "display_name": "Telegram Tool", "kind": "tool", - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "description": "Your agent uses your Telegram account to read and send messages", "keywords": [ "messaging", diff --git a/registry/tools/web-search.json b/registry/tools/web-search.json index 2a3f9a5d..9c9111ac 100644 --- a/registry/tools/web-search.json +++ b/registry/tools/web-search.json @@ -2,8 +2,8 @@ "name": "web-search", "display_name": "Web Search", "kind": "tool", - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "description": "Search the web using Brave Search API", "keywords": [ "search", diff --git a/src/agent/agent_loop.rs b/src/agent/agent_loop.rs index 0bf1fd58..60d0ea2e 100644 --- a/src/agent/agent_loop.rs +++ b/src/agent/agent_loop.rs @@ -77,6 +77,10 @@ pub struct AgentDeps { pub sse_tx: Option>, /// HTTP interceptor for trace recording/replay. pub http_interceptor: Option>, + /// Audio transcription middleware for voice messages. + pub transcription: Option>, + /// Document text extraction middleware for PDF, DOCX, PPTX, etc. + pub document_extraction: Option>, } /// The main agent that coordinates all components. @@ -524,6 +528,20 @@ impl Agent { } }; + // Apply transcription middleware to audio attachments + let mut message = message; + if let Some(ref transcription) = self.deps.transcription { + transcription.process(&mut message).await; + } + + // Apply document extraction middleware to document attachments + if let Some(ref doc_extraction) = self.deps.document_extraction { + doc_extraction.process(&mut message).await; + } + + // Store successfully extracted document text in workspace for indexing + self.store_extracted_documents(&message).await; + match self.handle_message(&message).await { Ok(Some(response)) if !response.is_empty() => { // Hook: BeforeOutbound — allow hooks to modify or suppress outbound @@ -622,6 +640,73 @@ impl Agent { Ok(()) } + /// Store extracted document text in workspace memory for future search/recall. + async fn store_extracted_documents(&self, message: &IncomingMessage) { + let workspace = match self.workspace() { + Some(ws) => ws, + None => return, + }; + + for attachment in &message.attachments { + if attachment.kind != crate::channels::AttachmentKind::Document { + continue; + } + let text = match &attachment.extracted_text { + Some(t) if !t.starts_with('[') => t, // skip error messages like "[Failed to..." + _ => continue, + }; + + // Sanitize filename: strip path separators to prevent directory traversal + let raw_name = attachment.filename.as_deref().unwrap_or("unnamed_document"); + let filename: String = raw_name + .chars() + .map(|c| { + if c == '/' || c == '\\' || c == '\0' { + '_' + } else { + c + } + }) + .collect(); + let filename = filename.trim_start_matches('.'); + let filename = if filename.is_empty() { + "unnamed_document" + } else { + filename + }; + let date = chrono::Utc::now().format("%Y-%m-%d"); + let path = format!("documents/{date}/{filename}"); + + let header = format!( + "# {filename}\n\n\ + > Uploaded by **{}** via **{}** on {date}\n\ + > MIME: {} | Size: {} bytes\n\n---\n\n", + message.user_id, + message.channel, + attachment.mime_type, + attachment.size_bytes.unwrap_or(0), + ); + let content = format!("{header}{text}"); + + match workspace.write(&path, &content).await { + Ok(_) => { + tracing::info!( + path = %path, + text_len = text.len(), + "Stored extracted document in workspace memory" + ); + } + Err(e) => { + tracing::warn!( + path = %path, + error = %e, + "Failed to store extracted document in workspace" + ); + } + } + } + } + async fn handle_message(&self, message: &IncomingMessage) -> Result, Error> { // Set message tool context for this turn (current channel and target) // For Signal, use signal_target from metadata (group:ID or phone number), diff --git a/src/agent/attachments.rs b/src/agent/attachments.rs new file mode 100644 index 00000000..cb522912 --- /dev/null +++ b/src/agent/attachments.rs @@ -0,0 +1,307 @@ +//! Augment user message content with structured attachment context. + +use base64::Engine; + +use crate::channels::{AttachmentKind, IncomingAttachment}; +use crate::llm::{ContentPart, ImageUrl}; + +/// Result of processing attachments for the LLM pipeline. +pub struct AugmentResult { + /// Augmented text content with attachment metadata appended. + pub text: String, + /// Image content parts to include as multimodal input. + pub image_parts: Vec, +} + +/// Process attachments into augmented text and multimodal image parts. +/// +/// Returns `None` if `attachments` is empty (caller should use original content). +/// Returns `Some(AugmentResult)` with: +/// - `text`: original content + `` block (metadata, transcripts, etc.) +/// - `image_parts`: `ContentPart::ImageUrl` entries for images with data +pub fn augment_with_attachments( + content: &str, + attachments: &[IncomingAttachment], +) -> Option { + if attachments.is_empty() { + return None; + } + + let mut text = content.to_string(); + text.push_str("\n\n"); + + let mut image_parts = Vec::new(); + + for (i, att) in attachments.iter().enumerate() { + text.push('\n'); + text.push_str(&format_attachment(i + 1, att)); + + // Build multimodal image part when image data is available + if att.kind == AttachmentKind::Image && !att.data.is_empty() { + let b64 = base64::engine::general_purpose::STANDARD.encode(&att.data); + let data_url = format!("data:{};base64,{}", att.mime_type, b64); + image_parts.push(ContentPart::ImageUrl { + image_url: ImageUrl { + url: data_url, + detail: None, + }, + }); + } + } + + text.push_str("\n"); + Some(AugmentResult { text, image_parts }) +} + +/// Escape a string for use as an XML attribute value. +fn escape_xml_attr(s: &str) -> String { + s.replace('&', "&") + .replace('"', """) + .replace('<', "<") + .replace('>', ">") +} + +/// Escape a string for use as XML text content. +fn escape_xml_text(s: &str) -> String { + s.replace('&', "&") + .replace('<', "<") + .replace('>', ">") +} + +fn format_attachment(index: usize, att: &IncomingAttachment) -> String { + let filename = escape_xml_attr(att.filename.as_deref().unwrap_or("unknown")); + let mime = escape_xml_attr(&att.mime_type); + + match &att.kind { + AttachmentKind::Audio => { + let duration_attr = att + .duration_secs + .map(|d| format!(" duration=\"{d}s\"")) + .unwrap_or_default(); + + let body = match &att.extracted_text { + Some(text) => format!("Transcript: {}", escape_xml_text(text)), + None => "Audio transcript unavailable.".to_string(), + }; + + format!( + "\n\ + {body}\n\ + " + ) + } + AttachmentKind::Image => { + let size_attr = att + .size_bytes + .map(|s| format!(" size=\"{}\"", format_size(s))) + .unwrap_or_default(); + + let body = if att.data.is_empty() { + "[Image attached — visual content not available in this conversation]" + } else { + "[Image attached — sent as visual content]" + }; + + format!( + "\n\ + {body}\n\ + " + ) + } + AttachmentKind::Document => { + let body: String = match &att.extracted_text { + Some(text) => escape_xml_text(text), + None => { + let size_info = att + .size_bytes + .map(|s| format!(" size=\"{}\"", format_size(s))) + .unwrap_or_default(); + return format!( + "\n\ + [Document attached — text extraction unavailable]\n\ + " + ); + } + }; + + let size_attr = att + .size_bytes + .map(|s| format!(" size=\"{}\"", format_size(s))) + .unwrap_or_default(); + + format!( + "\n\ + {body}\n\ + " + ) + } + } +} + +fn format_size(bytes: u64) -> String { + if bytes < 1024 { + format!("{bytes}B") + } else if bytes < 1024 * 1024 { + format!("{}KB", bytes / 1024) + } else { + format!("{:.1}MB", bytes as f64 / (1024.0 * 1024.0)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_attachment(kind: AttachmentKind) -> IncomingAttachment { + IncomingAttachment { + id: "test-id".to_string(), + kind, + mime_type: "application/octet-stream".to_string(), + filename: None, + size_bytes: None, + source_url: None, + storage_key: None, + extracted_text: None, + data: vec![], + duration_secs: None, + } + } + + #[test] + fn empty_attachments_returns_none() { + assert!(augment_with_attachments("hello", &[]).is_none()); + } + + #[test] + fn audio_with_transcript() { + let mut att = make_attachment(AttachmentKind::Audio); + att.filename = Some("voice.ogg".to_string()); + att.extracted_text = Some("Hello, can you help me?".to_string()); + att.duration_secs = Some(5); + + let result = augment_with_attachments("hi", &[att]).unwrap(); + assert!(result.text.starts_with("hi\n\n")); + assert!(result.text.contains("type=\"audio\"")); + assert!(result.text.contains("filename=\"voice.ogg\"")); + assert!(result.text.contains("duration=\"5s\"")); + assert!(result.text.contains("Transcript: Hello, can you help me?")); + assert!(result.text.ends_with("")); + assert!(result.image_parts.is_empty()); + } + + #[test] + fn audio_without_transcript() { + let mut att = make_attachment(AttachmentKind::Audio); + att.filename = Some("voice.ogg".to_string()); + att.duration_secs = Some(10); + + let result = augment_with_attachments("hi", &[att]).unwrap(); + assert!(result.text.contains("Audio transcript unavailable.")); + assert!(result.text.contains("duration=\"10s\"")); + } + + #[test] + fn image_without_data_no_visual() { + let mut att = make_attachment(AttachmentKind::Image); + att.filename = Some("screenshot.png".to_string()); + att.mime_type = "image/png".to_string(); + att.size_bytes = Some(245_000); + + let result = augment_with_attachments("check this", &[att]).unwrap(); + assert!(result.text.contains("type=\"image\"")); + assert!(result.text.contains("filename=\"screenshot.png\"")); + assert!(result.text.contains("mime=\"image/png\"")); + assert!(result.text.contains("size=\"239KB\"")); + assert!( + result + .text + .contains("[Image attached — visual content not available in this conversation]") + ); + assert!(result.image_parts.is_empty()); + } + + #[test] + fn image_with_data_produces_content_part() { + let mut att = make_attachment(AttachmentKind::Image); + att.filename = Some("photo.jpg".to_string()); + att.mime_type = "image/jpeg".to_string(); + att.data = vec![0xFF, 0xD8, 0xFF]; // fake JPEG header + + let result = augment_with_attachments("look", &[att]).unwrap(); + assert!( + result + .text + .contains("[Image attached — sent as visual content]") + ); + assert_eq!(result.image_parts.len(), 1); + match &result.image_parts[0] { + ContentPart::ImageUrl { image_url } => { + assert!(image_url.url.starts_with("data:image/jpeg;base64,")); + } + other => panic!("Expected ImageUrl, got: {:?}", other), + } + } + + #[test] + fn document_with_extracted_text() { + let mut att = make_attachment(AttachmentKind::Document); + att.filename = Some("report.pdf".to_string()); + att.extracted_text = Some("Executive summary: Q3 results".to_string()); + + let result = augment_with_attachments("review", &[att]).unwrap(); + assert!(result.text.contains("type=\"document\"")); + assert!(result.text.contains("filename=\"report.pdf\"")); + assert!(result.text.contains("Executive summary: Q3 results")); + } + + #[test] + fn document_without_extracted_text() { + let mut att = make_attachment(AttachmentKind::Document); + att.filename = Some("data.csv".to_string()); + att.mime_type = "text/csv".to_string(); + att.size_bytes = Some(1024); + + let result = augment_with_attachments("analyze", &[att]).unwrap(); + assert!(result.text.contains("type=\"document\"")); + assert!(result.text.contains("mime=\"text/csv\"")); + assert!( + result + .text + .contains("[Document attached — text extraction unavailable]") + ); + } + + #[test] + fn multiple_attachments_with_mixed_images() { + let mut audio = make_attachment(AttachmentKind::Audio); + audio.filename = Some("voice.ogg".to_string()); + audio.extracted_text = Some("Hello".to_string()); + + let mut image_with_data = make_attachment(AttachmentKind::Image); + image_with_data.filename = Some("photo.jpg".to_string()); + image_with_data.mime_type = "image/jpeg".to_string(); + image_with_data.data = vec![0xFF, 0xD8]; + + let mut image_no_data = make_attachment(AttachmentKind::Image); + image_no_data.filename = Some("remote.png".to_string()); + image_no_data.mime_type = "image/png".to_string(); + + let result = + augment_with_attachments("msg", &[audio, image_with_data, image_no_data]).unwrap(); + assert!(result.text.contains("index=\"1\"")); + assert!(result.text.contains("index=\"2\"")); + assert!(result.text.contains("index=\"3\"")); + // Only the image with data produces a content part + assert_eq!(result.image_parts.len(), 1); + } + + #[test] + fn original_content_preserved() { + let original = "Please help me with this task"; + let mut att = make_attachment(AttachmentKind::Audio); + att.extracted_text = Some("transcript".to_string()); + + let result = augment_with_attachments(original, &[att]).unwrap(); + assert!(result.text.starts_with(original)); + } +} diff --git a/src/agent/dispatcher.rs b/src/agent/dispatcher.rs index 538e2b3d..2834777a 100644 --- a/src/agent/dispatcher.rs +++ b/src/agent/dispatcher.rs @@ -1127,6 +1127,8 @@ mod tests { cost_guard: Arc::new(CostGuard::new(CostGuardConfig::default())), sse_tx: None, http_interceptor: None, + transcription: None, + document_extraction: None, }; Agent::new( @@ -1879,6 +1881,8 @@ mod tests { cost_guard: Arc::new(CostGuard::new(CostGuardConfig::default())), sse_tx: None, http_interceptor: None, + transcription: None, + document_extraction: None, }; Agent::new( @@ -1992,6 +1996,8 @@ mod tests { cost_guard: Arc::new(CostGuard::new(CostGuardConfig::default())), sse_tx: None, http_interceptor: None, + transcription: None, + document_extraction: None, }; Agent::new( diff --git a/src/agent/mod.rs b/src/agent/mod.rs index 1fbbc3bf..895a551a 100644 --- a/src/agent/mod.rs +++ b/src/agent/mod.rs @@ -11,6 +11,7 @@ //! - Context compaction for long conversations mod agent_loop; +mod attachments; mod commands; pub mod compaction; pub mod context_monitor; diff --git a/src/agent/session.rs b/src/agent/session.rs index 4c3dbd67..5dee8b47 100644 --- a/src/agent/session.rs +++ b/src/agent/session.rs @@ -320,7 +320,14 @@ impl Thread { pub fn messages(&self) -> Vec { let mut messages = Vec::new(); for turn in &self.turns { - messages.push(ChatMessage::user(&turn.user_input)); + if turn.image_content_parts.is_empty() { + messages.push(ChatMessage::user(&turn.user_input)); + } else { + messages.push(ChatMessage::user_with_parts( + &turn.user_input, + turn.image_content_parts.clone(), + )); + } if let Some(ref response) = turn.response { messages.push(ChatMessage::assistant(response)); } @@ -407,6 +414,11 @@ pub struct Turn { pub completed_at: Option>, /// Error message (if failed). pub error: Option, + /// Transient image content parts for multimodal LLM input. + /// Not serialized — images are only needed for the current LLM call. + /// The text description in `user_input` persists for compaction/context. + #[serde(skip)] + pub image_content_parts: Vec, } impl Turn { @@ -421,6 +433,7 @@ impl Turn { started_at: Utc::now(), completed_at: None, error: None, + image_content_parts: Vec::new(), } } @@ -429,6 +442,8 @@ impl Turn { self.response = Some(response.into()); self.state = TurnState::Completed; self.completed_at = Some(Utc::now()); + // Free image data — only needed for the initial LLM call, not subsequent turns + self.image_content_parts.clear(); } /// Fail this turn. @@ -436,12 +451,14 @@ impl Turn { self.error = Some(error.into()); self.state = TurnState::Failed; self.completed_at = Some(Utc::now()); + self.image_content_parts.clear(); } /// Interrupt this turn. pub fn interrupt(&mut self) { self.state = TurnState::Interrupted; self.completed_at = Some(Utc::now()); + self.image_content_parts.clear(); } /// Record a tool call. diff --git a/src/agent/thread_ops.rs b/src/agent/thread_ops.rs index bd1e5258..954c0f02 100644 --- a/src/agent/thread_ops.rs +++ b/src/agent/thread_ops.rs @@ -257,6 +257,14 @@ impl Agent { ); } + // Augment content with attachment context (transcripts, metadata, images) + let augmented = + crate::agent::attachments::augment_with_attachments(content, &message.attachments); + let (effective_content, image_parts) = match &augmented { + Some(result) => (result.text.as_str(), result.image_parts.clone()), + None => (content, Vec::new()), + }; + // Start the turn and get messages let turn_messages = { let mut sess = session.lock().await; @@ -264,12 +272,13 @@ impl Agent { .threads .get_mut(&thread_id) .ok_or_else(|| Error::from(crate::error::JobError::NotFound { id: thread_id }))?; - thread.start_turn(content); + let turn = thread.start_turn(effective_content); + turn.image_content_parts = image_parts; thread.messages() }; // Persist user message to DB immediately so it survives crashes - self.persist_user_message(thread_id, &message.user_id, content) + self.persist_user_message(thread_id, &message.user_id, effective_content) .await; // Send thinking status diff --git a/src/channels/channel.rs b/src/channels/channel.rs index 46fbc9ca..1160c411 100644 --- a/src/channels/channel.rs +++ b/src/channels/channel.rs @@ -10,6 +10,56 @@ use uuid::Uuid; use crate::error::ChannelError; +/// Kind of attachment carried on an incoming message. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AttachmentKind { + /// Audio content (voice notes, audio files). + Audio, + /// Image content (photos, screenshots). + Image, + /// Document content (PDFs, files). + Document, +} + +impl AttachmentKind { + /// Infer attachment kind from MIME type. + pub fn from_mime_type(mime: &str) -> Self { + let base = mime.split(';').next().unwrap_or(mime).trim(); + if base.starts_with("audio/") { + Self::Audio + } else if base.starts_with("image/") { + Self::Image + } else { + Self::Document + } + } +} + +/// A file or media attachment on an incoming message. +#[derive(Debug, Clone)] +pub struct IncomingAttachment { + /// Unique identifier within the channel (e.g., Telegram file_id). + pub id: String, + /// What kind of content this is. + pub kind: AttachmentKind, + /// MIME type (e.g., "image/jpeg", "audio/ogg", "application/pdf"). + pub mime_type: String, + /// Original filename, if known. + pub filename: Option, + /// File size in bytes, if known. + pub size_bytes: Option, + /// URL to download the file from the channel's API. + pub source_url: Option, + /// Opaque key for host-side storage (e.g., after download/caching). + pub storage_key: Option, + /// Extracted text content (e.g., OCR result, PDF text, audio transcript). + pub extracted_text: Option, + /// Raw file bytes (for small files downloaded by the channel). + pub data: Vec, + /// Duration in seconds (for audio/video). + pub duration_secs: Option, +} + /// A message received from an external channel. #[derive(Debug, Clone)] pub struct IncomingMessage { @@ -29,6 +79,8 @@ pub struct IncomingMessage { pub received_at: DateTime, /// Channel-specific metadata. pub metadata: serde_json::Value, + /// File or media attachments on this message. + pub attachments: Vec, } impl IncomingMessage { @@ -47,6 +99,7 @@ impl IncomingMessage { thread_id: None, received_at: Utc::now(), metadata: serde_json::Value::Null, + attachments: Vec::new(), } } @@ -67,6 +120,12 @@ impl IncomingMessage { self.user_name = Some(name.into()); self } + + /// Set attachments. + pub fn with_attachments(mut self, attachments: Vec) -> Self { + self.attachments = attachments; + self + } } /// Stream of incoming messages. diff --git a/src/channels/mod.rs b/src/channels/mod.rs index ad7320d3..095c96c1 100644 --- a/src/channels/mod.rs +++ b/src/channels/mod.rs @@ -36,7 +36,10 @@ pub mod wasm; pub mod web; mod webhook_server; -pub use channel::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate}; +pub use channel::{ + AttachmentKind, Channel, IncomingAttachment, IncomingMessage, MessageStream, OutgoingResponse, + StatusUpdate, +}; pub use http::HttpChannel; pub use manager::ChannelManager; pub use repl::ReplChannel; diff --git a/src/channels/wasm/host.rs b/src/channels/wasm/host.rs index 946d9c5d..9f09455f 100644 --- a/src/channels/wasm/host.rs +++ b/src/channels/wasm/host.rs @@ -5,6 +5,7 @@ //! - Workspace write access (scoped to channel namespace) //! - Rate limiting for message emission +use std::collections::HashMap; use std::time::{SystemTime, UNIX_EPOCH}; use crate::channels::wasm::capabilities::{ChannelCapabilities, EmitRateLimitConfig}; @@ -17,6 +18,52 @@ const MAX_EMITS_PER_EXECUTION: usize = 100; /// Maximum message content size (64 KB). const MAX_MESSAGE_CONTENT_SIZE: usize = 64 * 1024; +/// A file or media attachment on an incoming message. +#[derive(Debug, Clone)] +pub struct Attachment { + /// Unique identifier within the channel (e.g., Telegram file_id). + pub id: String, + /// MIME type (e.g., "image/jpeg", "audio/ogg", "application/pdf"). + pub mime_type: String, + /// Original filename, if known. + pub filename: Option, + /// File size in bytes, if known. + pub size_bytes: Option, + /// URL to download the file from the channel's API. + pub source_url: Option, + /// Opaque key for host-side storage (e.g., after download/caching). + pub storage_key: Option, + /// Extracted text content (e.g., OCR result, PDF text, audio transcript). + pub extracted_text: Option, + /// Raw file bytes (for small files downloaded by the channel). + pub data: Vec, + /// Duration in seconds (for audio/video). + pub duration_secs: Option, +} + +/// Maximum total attachment size per message (20 MB). +const MAX_ATTACHMENT_TOTAL_SIZE: u64 = 20 * 1024 * 1024; + +/// Maximum number of attachments per message. +const MAX_ATTACHMENTS_PER_MESSAGE: usize = 10; + +/// Allowed MIME type prefixes for attachments. +const ALLOWED_MIME_PREFIXES: &[&str] = &[ + "image/", + "audio/", + "video/", + "application/pdf", + "application/vnd.", + "application/msword", + "application/rtf", + "text/", + "application/json", + "application/zip", + "application/gzip", + "application/x-tar", + "application/octet-stream", +]; + /// A message emitted by a WASM channel to be sent to the agent. #[derive(Debug, Clone)] pub struct EmittedMessage { @@ -35,6 +82,9 @@ pub struct EmittedMessage { /// Channel-specific metadata as JSON string. pub metadata_json: String, + /// File or media attachments on this message. + pub attachments: Vec, + /// Timestamp when the message was emitted. pub emitted_at_millis: u64, } @@ -48,6 +98,7 @@ impl EmittedMessage { content: content.into(), thread_id: None, metadata_json: "{}".to_string(), + attachments: Vec::new(), emitted_at_millis: SystemTime::now() .duration_since(UNIX_EPOCH) .map(|d| d.as_millis() as u64) @@ -72,6 +123,12 @@ impl EmittedMessage { self.metadata_json = metadata_json.into(); self } + + /// Set attachments. + pub fn with_attachments(mut self, attachments: Vec) -> Self { + self.attachments = attachments; + self + } } /// A pending workspace write operation. @@ -112,6 +169,13 @@ pub struct ChannelHostState { /// Count of emits dropped due to rate limiting. emits_dropped: usize, + + /// Binary data stored for attachments via `store-attachment-data`. + /// Keyed by attachment ID, cleared after callback completes. + attachment_data: HashMap>, + + /// Total bytes stored in attachment_data (for enforcing limits). + attachment_data_total: u64, } impl std::fmt::Debug for ChannelHostState { @@ -141,6 +205,8 @@ impl ChannelHostState { emit_count: 0, emit_enabled: true, emits_dropped: 0, + attachment_data: HashMap::new(), + attachment_data_total: 0, } } @@ -168,6 +234,7 @@ impl ChannelHostState { /// /// Messages are queued and delivered after callback execution completes. /// Rate limiting is enforced per-execution and globally. + /// Attachments are validated for count, total size, and MIME type. pub fn emit_message(&mut self, msg: EmittedMessage) -> Result<(), WasmChannelError> { // Check per-execution limit if !self.emit_enabled { @@ -186,6 +253,9 @@ impl ChannelHostState { return Ok(()); } + // Validate attachments + let msg = self.validate_attachments(msg); + // Validate message content size if msg.content.len() > MAX_MESSAGE_CONTENT_SIZE { tracing::warn!( @@ -209,6 +279,71 @@ impl ChannelHostState { Ok(()) } + /// Validate and sanitize attachments on an emitted message. + /// + /// Enforces count limits, total size limits, and MIME type allowlist. + /// Invalid attachments are dropped with a warning. + fn validate_attachments(&self, mut msg: EmittedMessage) -> EmittedMessage { + if msg.attachments.is_empty() { + return msg; + } + + // Enforce attachment count limit + if msg.attachments.len() > MAX_ATTACHMENTS_PER_MESSAGE { + tracing::warn!( + channel = %self.channel_name, + count = msg.attachments.len(), + max = MAX_ATTACHMENTS_PER_MESSAGE, + "Too many attachments, truncating" + ); + msg.attachments.truncate(MAX_ATTACHMENTS_PER_MESSAGE); + } + + // Filter by MIME type and enforce total size limit + let mut total_size: u64 = 0; + msg.attachments.retain(|att| { + let mime_ok = ALLOWED_MIME_PREFIXES + .iter() + .any(|prefix| att.mime_type.starts_with(prefix)); + if !mime_ok { + tracing::warn!( + channel = %self.channel_name, + mime_type = %att.mime_type, + "Attachment MIME type not allowed, dropping" + ); + return false; + } + + // Use the larger of reported size_bytes and actual stored data size + // to prevent WASM channels from under-reporting to bypass limits. + let stored_size = self + .attachment_data + .get(&att.id) + .map(|d| d.len() as u64) + .unwrap_or(att.data.len() as u64); + let size = att + .size_bytes + .map(|reported| reported.max(stored_size)) + .unwrap_or(stored_size); + if size > 0 { + total_size = total_size.saturating_add(size); + if total_size > MAX_ATTACHMENT_TOTAL_SIZE { + tracing::warn!( + channel = %self.channel_name, + total_size, + max = MAX_ATTACHMENT_TOTAL_SIZE, + "Attachment total size exceeded, dropping" + ); + return false; + } + } + + true + }); + + msg + } + /// Take all emitted messages (clears the queue). pub fn take_emitted_messages(&mut self) -> Vec { std::mem::take(&mut self.emitted_messages) @@ -224,6 +359,69 @@ impl ChannelHostState { self.emits_dropped } + /// Store binary data for an attachment. + /// + /// Called by WASM channels to associate downloaded bytes with an attachment ID. + /// The data is retrieved after callback completion and merged into `Attachment::data`. + pub fn store_attachment_data( + &mut self, + attachment_id: &str, + data: Vec, + ) -> Result<(), WasmChannelError> { + const MAX_PER_ATTACHMENT: u64 = 20 * 1024 * 1024; // 20 MB + const MAX_TOTAL: u64 = 50 * 1024 * 1024; // 50 MB + + let size = data.len() as u64; + if size > MAX_PER_ATTACHMENT { + return Err(WasmChannelError::CallbackFailed { + name: self.channel_name.clone(), + reason: format!( + "Attachment data too large: {} bytes (max {})", + size, MAX_PER_ATTACHMENT + ), + }); + } + + // Subtract the old entry size (if overwriting) before adding new size + let old_size = self + .attachment_data + .get(attachment_id) + .map(|d| d.len() as u64) + .unwrap_or(0); + let adjusted_total = self.attachment_data_total.saturating_sub(old_size); + let new_total = adjusted_total.saturating_add(size); + if new_total > MAX_TOTAL { + return Err(WasmChannelError::CallbackFailed { + name: self.channel_name.clone(), + reason: format!( + "Total attachment data too large: {} bytes (max {})", + new_total, MAX_TOTAL + ), + }); + } + + self.attachment_data_total = new_total; + self.attachment_data.insert(attachment_id.to_string(), data); + Ok(()) + } + + /// Remove stored binary data for a specific attachment ID. + pub fn remove_attachment_data(&mut self, id: &str) -> Option> { + if let Some(data) = self.attachment_data.remove(id) { + self.attachment_data_total = + self.attachment_data_total.saturating_sub(data.len() as u64); + Some(data) + } else { + None + } + } + + /// Take all stored attachment data (clears the store). + pub fn take_attachment_data(&mut self) -> HashMap> { + self.attachment_data_total = 0; + std::mem::take(&mut self.attachment_data) + } + /// Write to workspace (scoped to channel namespace). /// /// Writes are queued and committed after callback execution completes. @@ -431,7 +629,8 @@ impl ChannelEmitRateLimiter { mod tests { use crate::channels::wasm::capabilities::{ChannelCapabilities, EmitRateLimitConfig}; use crate::channels::wasm::host::{ - ChannelEmitRateLimiter, ChannelHostState, EmittedMessage, MAX_EMITS_PER_EXECUTION, + Attachment, ChannelEmitRateLimiter, ChannelHostState, EmittedMessage, + MAX_ATTACHMENT_TOTAL_SIZE, MAX_ATTACHMENTS_PER_MESSAGE, MAX_EMITS_PER_EXECUTION, }; #[test] @@ -760,4 +959,133 @@ mod tests { Some("200".to_string()) ); } + + // === Attachment validation tests === + + fn make_attachment(id: &str, mime: &str, size: Option) -> Attachment { + Attachment { + id: id.to_string(), + mime_type: mime.to_string(), + filename: None, + size_bytes: size, + source_url: None, + storage_key: None, + extracted_text: None, + data: Vec::new(), + duration_secs: None, + } + } + + #[test] + fn test_emit_message_with_attachments() { + let caps = ChannelCapabilities::for_channel("test"); + let mut state = ChannelHostState::new("test", caps); + + let msg = EmittedMessage::new("user1", "Check this image") + .with_attachments(vec![make_attachment("file1", "image/jpeg", Some(1024))]); + + state.emit_message(msg).unwrap(); + + let messages = state.take_emitted_messages(); + assert_eq!(messages.len(), 1); + assert_eq!(messages[0].attachments.len(), 1); + assert_eq!(messages[0].attachments[0].id, "file1"); + assert_eq!(messages[0].attachments[0].mime_type, "image/jpeg"); + assert_eq!(messages[0].attachments[0].size_bytes, Some(1024)); + } + + #[test] + fn test_emit_message_no_attachments_backward_compat() { + let caps = ChannelCapabilities::for_channel("test"); + let mut state = ChannelHostState::new("test", caps); + + let msg = EmittedMessage::new("user1", "Just text"); + state.emit_message(msg).unwrap(); + + let messages = state.take_emitted_messages(); + assert_eq!(messages.len(), 1); + assert!(messages[0].attachments.is_empty()); + } + + #[test] + fn test_attachment_count_limit() { + let caps = ChannelCapabilities::for_channel("test"); + let mut state = ChannelHostState::new("test", caps); + + let attachments: Vec = (0..MAX_ATTACHMENTS_PER_MESSAGE + 5) + .map(|i| make_attachment(&format!("file{}", i), "image/png", Some(100))) + .collect(); + + let msg = EmittedMessage::new("user1", "Many files").with_attachments(attachments); + state.emit_message(msg).unwrap(); + + let messages = state.take_emitted_messages(); + assert_eq!(messages[0].attachments.len(), MAX_ATTACHMENTS_PER_MESSAGE); + } + + #[test] + fn test_attachment_total_size_limit() { + let caps = ChannelCapabilities::for_channel("test"); + let mut state = ChannelHostState::new("test", caps); + + // Each file is 1/3 of the limit, so 3 fit but 4th does not + let chunk_size = MAX_ATTACHMENT_TOTAL_SIZE / 3; + let attachments = vec![ + make_attachment("file1", "image/png", Some(chunk_size)), + make_attachment("file2", "image/png", Some(chunk_size)), + make_attachment("file3", "image/png", Some(chunk_size)), + make_attachment("file4", "image/png", Some(chunk_size)), + ]; + + let msg = EmittedMessage::new("user1", "Big files").with_attachments(attachments); + state.emit_message(msg).unwrap(); + + let messages = state.take_emitted_messages(); + // Only first 3 fit within the total size limit + assert_eq!(messages[0].attachments.len(), 3); + } + + #[test] + fn test_attachment_mime_type_filtering() { + let caps = ChannelCapabilities::for_channel("test"); + let mut state = ChannelHostState::new("test", caps); + + let attachments = vec![ + make_attachment("ok1", "image/jpeg", Some(100)), + make_attachment("bad1", "application/x-executable", Some(100)), + make_attachment("ok2", "application/pdf", Some(100)), + make_attachment("bad2", "application/x-msdos-program", Some(100)), + make_attachment("ok3", "text/plain", Some(100)), + make_attachment("ok4", "audio/mpeg", Some(100)), + make_attachment("ok5", "video/mp4", Some(100)), + ]; + + let msg = EmittedMessage::new("user1", "Mixed files").with_attachments(attachments); + state.emit_message(msg).unwrap(); + + let messages = state.take_emitted_messages(); + let ids: Vec<&str> = messages[0] + .attachments + .iter() + .map(|a| a.id.as_str()) + .collect(); + assert_eq!(ids, vec!["ok1", "ok2", "ok3", "ok4", "ok5"]); + } + + #[test] + fn test_attachment_unknown_size_allowed() { + let caps = ChannelCapabilities::for_channel("test"); + let mut state = ChannelHostState::new("test", caps); + + let attachments = vec![ + make_attachment("file1", "image/jpeg", None), + make_attachment("file2", "image/png", None), + ]; + + let msg = EmittedMessage::new("user1", "No sizes").with_attachments(attachments); + state.emit_message(msg).unwrap(); + + let messages = state.take_emitted_messages(); + assert_eq!(messages[0].attachments.len(), 2); + } } diff --git a/src/channels/wasm/wrapper.rs b/src/channels/wasm/wrapper.rs index 28272769..cac0cb1f 100644 --- a/src/channels/wasm/wrapper.rs +++ b/src/channels/wasm/wrapper.rs @@ -532,9 +532,45 @@ impl near::agent::channel_host::Host for ChannelStoreData { user_id = %msg.user_id, user_name = ?msg.user_name, content_len = msg.content.len(), + attachment_count = msg.attachments.len(), "WASM emit_message called" ); + let attachments: Vec = msg + .attachments + .into_iter() + .map(|a| { + // Parse extras-json for well-known fields + let extras: serde_json::Value = if a.extras_json.is_empty() { + serde_json::Value::Null + } else { + serde_json::from_str(&a.extras_json).unwrap_or(serde_json::Value::Null) + }; + let duration_secs = extras + .get("duration_secs") + .and_then(|v| v.as_u64()) + .map(|v| v as u32); + + // Merge stored binary data (from store-attachment-data host call) + let data = self + .host_state + .remove_attachment_data(&a.id) + .unwrap_or_default(); + + crate::channels::wasm::host::Attachment { + id: a.id, + mime_type: a.mime_type, + filename: a.filename, + size_bytes: a.size_bytes, + source_url: a.source_url, + storage_key: a.storage_key, + extracted_text: a.extracted_text, + data, + duration_secs, + } + }) + .collect(); + let mut emitted = EmittedMessage::new(msg.user_id.clone(), msg.content.clone()); if let Some(name) = msg.user_name { emitted = emitted.with_user_name(name); @@ -543,6 +579,7 @@ impl near::agent::channel_host::Host for ChannelStoreData { emitted = emitted.with_thread_id(tid); } emitted = emitted.with_metadata(msg.metadata_json); + emitted = emitted.with_attachments(attachments); match self.host_state.emit_message(emitted) { Ok(()) => { @@ -554,6 +591,21 @@ impl near::agent::channel_host::Host for ChannelStoreData { } } + fn store_attachment_data( + &mut self, + attachment_id: String, + data: Vec, + ) -> Result<(), String> { + tracing::debug!( + attachment_id = %attachment_id, + size = data.len(), + "WASM store_attachment_data called" + ); + self.host_state + .store_attachment_data(&attachment_id, data) + .map_err(|e| e.to_string()) + } + fn pairing_upsert_request( &mut self, channel: String, @@ -1327,12 +1379,14 @@ impl WasmChannel { content: &str, thread_id: Option<&str>, metadata_json: &str, + attachments: &[String], ) -> Result<(), WasmChannelError> { tracing::info!( channel = %self.name, message_id = %message_id, content_len = content.len(), thread_id = ?thread_id, + attachment_count = attachments.len(), "call_on_respond invoked" ); @@ -1370,12 +1424,21 @@ impl WasmChannel { let content = content.to_string(); let thread_id = thread_id.map(|s| s.to_string()); let metadata_json = metadata_json.to_string(); + let attachments = attachments.to_vec(); // Execute in blocking task with timeout tracing::info!(channel = %channel_name, "Starting on_respond WASM execution"); let result = tokio::time::timeout(timeout, async move { tokio::task::spawn_blocking(move || { + // Read attachment files from disk before entering WASM + let wit_attachments = read_attachments(&attachments).map_err(|e| { + WasmChannelError::CallbackFailed { + name: prepared.name.clone(), + reason: e, + } + })?; + tracing::info!("Creating WASM store for on_respond"); let mut store = Self::create_store( &runtime, @@ -1395,6 +1458,7 @@ impl WasmChannel { content: content.clone(), thread_id, metadata_json, + attachments: wit_attachments, }; // Truncate at char boundary for logging (avoid panic on multi-byte UTF-8) @@ -1458,6 +1522,124 @@ impl WasmChannel { } } + /// Execute the on_broadcast callback. + /// + /// Called to send a proactive message to a user without a prior incoming message. + pub async fn call_on_broadcast( + &self, + user_id: &str, + content: &str, + thread_id: Option<&str>, + attachments: &[String], + ) -> Result<(), WasmChannelError> { + tracing::info!( + channel = %self.name, + user_id = %user_id, + content_len = content.len(), + attachment_count = attachments.len(), + "call_on_broadcast invoked" + ); + + // If no WASM bytes, do nothing (for testing) + if self.prepared.component().is_none() { + tracing::debug!( + channel = %self.name, + "WASM channel on_broadcast called (no WASM module)" + ); + return Ok(()); + } + + let runtime = Arc::clone(&self.runtime); + let prepared = Arc::clone(&self.prepared); + let capabilities = self.capabilities.clone(); + let timeout = self.runtime.config().callback_timeout; + let channel_name = self.name.clone(); + let credentials = self.get_credentials().await; + let host_credentials = + resolve_channel_host_credentials(&self.capabilities, self.secrets_store.as_deref()) + .await; + let pairing_store = self.pairing_store.clone(); + + let user_id = user_id.to_string(); + let content = content.to_string(); + let thread_id = thread_id.map(|s| s.to_string()); + let attachments = attachments.to_vec(); + + let result = tokio::time::timeout(timeout, async move { + tokio::task::spawn_blocking(move || { + // Read attachment files from disk + let wit_attachments = read_attachments(&attachments).map_err(|e| { + WasmChannelError::CallbackFailed { + name: prepared.name.clone(), + reason: e, + } + })?; + + let mut store = Self::create_store( + &runtime, + &prepared, + &capabilities, + credentials, + host_credentials, + pairing_store, + )?; + + let instance = Self::instantiate_component(&runtime, &prepared, &mut store)?; + + let wit_response = wit_channel::AgentResponse { + message_id: String::new(), + content: content.clone(), + thread_id, + metadata_json: String::new(), + attachments: wit_attachments, + }; + + let channel_iface = instance.near_agent_channel(); + let wasm_result = channel_iface + .call_on_broadcast(&mut store, &user_id, &wit_response) + .map_err(|e| { + tracing::error!(error = %e, "WASM on_broadcast call failed"); + Self::map_wasm_error(e, &prepared.name, prepared.limits.fuel) + })?; + + if let Err(ref err_msg) = wasm_result { + tracing::error!(error = %err_msg, "WASM on_broadcast returned error"); + return Err(WasmChannelError::CallbackFailed { + name: prepared.name.clone(), + reason: err_msg.clone(), + }); + } + + let host_state = + Self::extract_host_state(&mut store, &prepared.name, &capabilities); + tracing::info!("on_broadcast WASM execution completed successfully"); + Ok(((), host_state)) + }) + .await + .map_err(|e| WasmChannelError::ExecutionPanicked { + name: channel_name.clone(), + reason: e.to_string(), + })? + }) + .await; + + let channel_name = self.name.clone(); + match result { + Ok(Ok(((), _host_state))) => { + tracing::debug!( + channel = %channel_name, + "WASM channel on_broadcast completed" + ); + Ok(()) + } + Ok(Err(e)) => Err(e), + Err(_) => Err(WasmChannelError::Timeout { + name: channel_name, + callback: "on_broadcast".to_string(), + }), + } + } + /// Execute the on_status callback. /// /// Called to notify the WASM channel of agent status changes (e.g., typing). @@ -1745,7 +1927,7 @@ impl WasmChannel { let metadata_json = serde_json::to_string(metadata).unwrap_or_default(); if let Err(e) = self - .call_on_respond(uuid::Uuid::new_v4(), &prompt, None, &metadata_json) + .call_on_respond(uuid::Uuid::new_v4(), &prompt, None, &metadata_json, &[]) .await { tracing::warn!( @@ -1847,6 +2029,27 @@ impl WasmChannel { msg = msg.with_thread(thread_id); } + // Convert attachments + if !emitted.attachments.is_empty() { + let incoming_attachments = emitted + .attachments + .iter() + .map(|a| crate::channels::IncomingAttachment { + id: a.id.clone(), + kind: crate::channels::AttachmentKind::from_mime_type(&a.mime_type), + mime_type: a.mime_type.clone(), + filename: a.filename.clone(), + size_bytes: a.size_bytes, + source_url: a.source_url.clone(), + storage_key: a.storage_key.clone(), + extracted_text: a.extracted_text.clone(), + data: a.data.clone(), + duration_secs: a.duration_secs, + }) + .collect(); + msg = msg.with_attachments(incoming_attachments); + } + // Parse metadata JSON if let Ok(metadata) = serde_json::from_str(&emitted.metadata_json) { msg = msg.with_metadata(metadata); @@ -1859,6 +2062,7 @@ impl WasmChannel { channel = %self.name, user_id = %emitted.user_id, content_len = emitted.content.len(), + attachment_count = msg.attachments.len(), "Sending emitted message to agent" ); @@ -2112,6 +2316,27 @@ impl WasmChannel { msg = msg.with_thread(thread_id); } + // Convert attachments + if !emitted.attachments.is_empty() { + let incoming_attachments = emitted + .attachments + .iter() + .map(|a| crate::channels::IncomingAttachment { + id: a.id.clone(), + kind: crate::channels::AttachmentKind::from_mime_type(&a.mime_type), + mime_type: a.mime_type.clone(), + filename: a.filename.clone(), + size_bytes: a.size_bytes, + source_url: a.source_url.clone(), + storage_key: a.storage_key.clone(), + extracted_text: a.extracted_text.clone(), + data: a.data.clone(), + duration_secs: a.duration_secs, + }) + .collect(); + msg = msg.with_attachments(incoming_attachments); + } + // Parse metadata JSON if let Ok(metadata) = serde_json::from_str(&emitted.metadata_json) { msg = msg.with_metadata(metadata); @@ -2130,6 +2355,7 @@ impl WasmChannel { channel = %channel_name, user_id = %emitted.user_id, content_len = emitted.content.len(), + attachment_count = msg.attachments.len(), "Sending polled message to agent" ); @@ -2257,6 +2483,7 @@ impl Channel for WasmChannel { &response.content, response.thread_id.as_deref(), &metadata_json, + &response.attachments, ) .await .map_err(|e| ChannelError::SendFailed { @@ -2269,24 +2496,15 @@ impl Channel for WasmChannel { async fn broadcast( &self, - _user_id: &str, + user_id: &str, response: OutgoingResponse, ) -> Result<(), ChannelError> { - let metadata_json = self - .last_broadcast_metadata - .read() - .await - .clone() - .ok_or_else(|| ChannelError::SendFailed { - name: self.name.clone(), - reason: "No messages received yet — no chat_id available for broadcast".into(), - })?; - - self.call_on_respond( - uuid::Uuid::new_v4(), + self.cancel_typing_task().await; + self.call_on_broadcast( + user_id, &response.content, response.thread_id.as_deref(), - &metadata_json, + &response.attachments, ) .await .map_err(|e| ChannelError::SendFailed { @@ -2749,6 +2967,79 @@ async fn resolve_channel_host_credentials( resolved } +// ============================================================================ +// Attachment Helpers +// ============================================================================ + +/// Maximum total attachment size (50 MB). +const MAX_TOTAL_ATTACHMENT_BYTES: u64 = 50 * 1024 * 1024; + +/// Detect MIME type from file extension using the `mime_guess` crate. +fn mime_from_extension(path: &str) -> String { + mime_guess::from_path(path) + .first_or_octet_stream() + .to_string() +} + +/// Read attachment files from disk and build WIT attachment records. +/// +/// Validates total size against `MAX_TOTAL_ATTACHMENT_BYTES`. +fn read_attachments(paths: &[String]) -> Result, String> { + if paths.is_empty() { + return Ok(Vec::new()); + } + + let mut attachments = Vec::with_capacity(paths.len()); + let mut total_bytes: u64 = 0; + let tmp_base = std::path::Path::new("/tmp"); + let home_base = dirs::home_dir() + .map(|h| h.join(".ironclaw")) + .unwrap_or_default(); + + for path in paths { + // Validate paths are under /tmp/ or ~/.ironclaw/ to prevent arbitrary file reads + let validated = crate::tools::builtin::path_utils::validate_path(path, Some(tmp_base)) + .or_else(|_| crate::tools::builtin::path_utils::validate_path(path, Some(&home_base))); + let validated = validated.map_err(|e| { + format!( + "Invalid attachment path '{}': must be under /tmp/ or ~/.ironclaw/: {}", + path, e + ) + })?; + + // Pre-check file size before reading into memory to avoid OOM + let file_size = std::fs::metadata(&validated) + .map_err(|e| format!("Failed to stat attachment '{}': {}", validated.display(), e))? + .len(); + total_bytes += file_size; + if total_bytes > MAX_TOTAL_ATTACHMENT_BYTES { + return Err(format!( + "Total attachment size exceeds {} MB limit", + MAX_TOTAL_ATTACHMENT_BYTES / (1024 * 1024) + )); + } + + let data = std::fs::read(&validated) + .map_err(|e| format!("Failed to read attachment '{}': {}", validated.display(), e))?; + + let filename = validated + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("file") + .to_string(); + + let mime_type = mime_from_extension(path); + + attachments.push(wit_channel::Attachment { + filename, + mime_type, + data, + }); + } + + Ok(attachments) +} + #[cfg(test)] mod tests { use std::sync::Arc; @@ -3871,4 +4162,139 @@ mod tests { // 404 because "000" is not a valid bot token assert_eq!(result, 404); } + + #[tokio::test] + async fn test_dispatch_emitted_messages_preserves_attachments() { + use crate::channels::wasm::host::{Attachment, EmittedMessage}; + + let (tx, mut rx) = tokio::sync::mpsc::channel(10); + let message_tx = Arc::new(tokio::sync::RwLock::new(Some(tx))); + + let rate_limiter = Arc::new(tokio::sync::RwLock::new( + crate::channels::wasm::host::ChannelEmitRateLimiter::new( + crate::channels::wasm::capabilities::EmitRateLimitConfig::default(), + ), + )); + + let attachments = vec![ + Attachment { + id: "photo123".to_string(), + mime_type: "image/jpeg".to_string(), + filename: Some("cat.jpg".to_string()), + size_bytes: Some(50_000), + source_url: Some("https://api.telegram.org/file/photo123".to_string()), + storage_key: None, + extracted_text: None, + data: Vec::new(), + duration_secs: None, + }, + Attachment { + id: "doc456".to_string(), + mime_type: "application/pdf".to_string(), + filename: Some("report.pdf".to_string()), + size_bytes: Some(120_000), + source_url: None, + storage_key: Some("store/doc456".to_string()), + extracted_text: Some("Report contents...".to_string()), + data: Vec::new(), + duration_secs: None, + }, + ]; + + let messages = + vec![EmittedMessage::new("user1", "Check these files").with_attachments(attachments)]; + + let last_broadcast_metadata = Arc::new(tokio::sync::RwLock::new(None)); + let result = WasmChannel::dispatch_emitted_messages( + "test-channel", + messages, + &message_tx, + &rate_limiter, + &last_broadcast_metadata, + None, + ) + .await; + + assert!(result.is_ok()); + + let msg = rx.try_recv().expect("Should receive message"); + assert_eq!(msg.content, "Check these files"); + assert_eq!(msg.attachments.len(), 2); + + // Verify first attachment + assert_eq!(msg.attachments[0].id, "photo123"); + assert_eq!(msg.attachments[0].mime_type, "image/jpeg"); + assert_eq!(msg.attachments[0].filename, Some("cat.jpg".to_string())); + assert_eq!(msg.attachments[0].size_bytes, Some(50_000)); + assert_eq!( + msg.attachments[0].source_url, + Some("https://api.telegram.org/file/photo123".to_string()) + ); + + // Verify second attachment + assert_eq!(msg.attachments[1].id, "doc456"); + assert_eq!(msg.attachments[1].mime_type, "application/pdf"); + assert_eq!( + msg.attachments[1].extracted_text, + Some("Report contents...".to_string()) + ); + assert_eq!( + msg.attachments[1].storage_key, + Some("store/doc456".to_string()) + ); + } + + #[tokio::test] + async fn test_dispatch_emitted_messages_no_attachments_backward_compat() { + use crate::channels::wasm::host::EmittedMessage; + + let (tx, mut rx) = tokio::sync::mpsc::channel(10); + let message_tx = Arc::new(tokio::sync::RwLock::new(Some(tx))); + + let rate_limiter = Arc::new(tokio::sync::RwLock::new( + crate::channels::wasm::host::ChannelEmitRateLimiter::new( + crate::channels::wasm::capabilities::EmitRateLimitConfig::default(), + ), + )); + + let messages = vec![EmittedMessage::new("user1", "Just text, no attachments")]; + + let last_broadcast_metadata = Arc::new(tokio::sync::RwLock::new(None)); + let result = WasmChannel::dispatch_emitted_messages( + "test-channel", + messages, + &message_tx, + &rate_limiter, + &last_broadcast_metadata, + None, + ) + .await; + + assert!(result.is_ok()); + + let msg = rx.try_recv().expect("Should receive message"); + assert_eq!(msg.content, "Just text, no attachments"); + assert!(msg.attachments.is_empty()); + } + + #[test] + fn test_mime_from_extension() { + use super::mime_from_extension; + assert_eq!(mime_from_extension("screenshot.png"), "image/png"); + assert_eq!(mime_from_extension("photo.JPG"), "image/jpeg"); + assert_eq!(mime_from_extension("photo.jpeg"), "image/jpeg"); + assert_eq!(mime_from_extension("animation.gif"), "image/gif"); + assert_eq!(mime_from_extension("doc.pdf"), "application/pdf"); + assert_eq!(mime_from_extension("video.mp4"), "video/mp4"); + assert_eq!(mime_from_extension("data.csv"), "text/csv"); + assert_eq!( + mime_from_extension("unknown.qqqzzz"), + "application/octet-stream" + ); + assert_eq!(mime_from_extension("noext"), "application/octet-stream"); + assert_eq!( + mime_from_extension("/home/user/.ironclaw/screenshot.png"), + "image/png" + ); + } } diff --git a/src/channels/web/handlers/extensions.rs b/src/channels/web/handlers/extensions.rs index 0c1f2905..078af7dc 100644 --- a/src/channels/web/handlers/extensions.rs +++ b/src/channels/web/handlers/extensions.rs @@ -62,6 +62,7 @@ pub async fn extensions_list_handler( has_auth: ext.has_auth, activation_status, activation_error: ext.activation_error, + version: ext.version, } }) .collect(); diff --git a/src/channels/web/openai_compat.rs b/src/channels/web/openai_compat.rs index c493ef5c..e329693a 100644 --- a/src/channels/web/openai_compat.rs +++ b/src/channels/web/openai_compat.rs @@ -244,6 +244,7 @@ pub fn convert_messages(messages: &[OpenAiMessage]) -> Result, _ => Ok(ChatMessage { role, content: m.content.as_deref().unwrap_or("").to_string(), + content_parts: Vec::new(), tool_call_id: None, name: m.name.clone(), tool_calls: None, diff --git a/src/channels/web/server.rs b/src/channels/web/server.rs index e456febd..9628bb2c 100644 --- a/src/channels/web/server.rs +++ b/src/channels/web/server.rs @@ -1438,6 +1438,7 @@ async fn extensions_list_handler( has_auth: ext.has_auth, activation_status, activation_error: ext.activation_error, + version: ext.version, } }) .collect(); @@ -1731,6 +1732,7 @@ async fn extensions_registry_handler( kind: kind_str, description: e.description.clone(), keywords: e.keywords.clone(), + version: e.version.clone(), } }) .collect(); diff --git a/src/channels/web/static/app.js b/src/channels/web/static/app.js index 0b69662d..84bb697e 100644 --- a/src/channels/web/static/app.js +++ b/src/channels/web/static/app.js @@ -1889,6 +1889,13 @@ function renderAvailableExtensionCard(entry) { kind.textContent = kindLabels[entry.kind] || entry.kind; header.appendChild(kind); + if (entry.version) { + const ver = document.createElement('span'); + ver.className = 'ext-version'; + ver.textContent = 'v' + entry.version; + header.appendChild(ver); + } + card.appendChild(header); const desc = document.createElement('div'); @@ -2049,6 +2056,13 @@ function renderExtensionCard(ext) { kind.textContent = kindLabels[ext.kind] || ext.kind; header.appendChild(kind); + if (ext.version) { + const ver = document.createElement('span'); + ver.className = 'ext-version'; + ver.textContent = 'v' + ext.version; + header.appendChild(ver); + } + // Auth dot only for non-WASM-channel extensions (channels use the stepper instead) if (ext.kind !== 'wasm_channel') { const authDot = document.createElement('span'); diff --git a/src/channels/web/static/style.css b/src/channels/web/static/style.css index ead9cec8..2889087d 100644 --- a/src/channels/web/static/style.css +++ b/src/channels/web/static/style.css @@ -2438,6 +2438,12 @@ body { color: var(--warning); } +.ext-version { + font-size: 11px; + color: var(--text-muted); + font-family: var(--font-mono); +} + .ext-auth-dot { width: 8px; height: 8px; diff --git a/src/channels/web/types.rs b/src/channels/web/types.rs index 0e74e26e..18610d87 100644 --- a/src/channels/web/types.rs +++ b/src/channels/web/types.rs @@ -401,6 +401,9 @@ pub struct ExtensionInfo { /// Human-readable error when activation_status is "failed". #[serde(skip_serializing_if = "Option::is_none")] pub activation_error: Option, + /// Extension version (semver). + #[serde(skip_serializing_if = "Option::is_none")] + pub version: Option, } #[derive(Debug, Serialize)] @@ -503,6 +506,8 @@ pub struct RegistryEntryInfo { pub description: String, pub keywords: Vec, pub installed: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub version: Option, } #[derive(Debug, Serialize)] diff --git a/src/config/mod.rs b/src/config/mod.rs index fab50b3e..74099ed8 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -19,6 +19,7 @@ mod safety; mod sandbox; mod secrets; mod skills; +mod transcription; mod tunnel; mod wasm; @@ -42,6 +43,7 @@ pub use self::safety::SafetyConfig; pub use self::sandbox::{ClaudeCodeConfig, SandboxModeConfig}; pub use self::secrets::SecretsConfig; pub use self::skills::SkillsConfig; +pub use self::transcription::TranscriptionConfig; pub use self::tunnel::TunnelConfig; pub use self::wasm::WasmConfig; pub use crate::llm::session::SessionConfig; @@ -72,6 +74,7 @@ pub struct Config { pub sandbox: SandboxModeConfig, pub claude_code: ClaudeCodeConfig, pub skills: SkillsConfig, + pub transcription: TranscriptionConfig, pub observability: crate::observability::ObservabilityConfig, } @@ -143,6 +146,7 @@ impl Config { installed_dir: installed_skills_dir, ..SkillsConfig::default() }, + transcription: TranscriptionConfig::default(), observability: crate::observability::ObservabilityConfig::default(), } } @@ -267,6 +271,7 @@ impl Config { sandbox: SandboxModeConfig::resolve()?, claude_code: ClaudeCodeConfig::resolve()?, skills: SkillsConfig::resolve()?, + transcription: TranscriptionConfig::resolve(settings)?, observability: crate::observability::ObservabilityConfig { backend: std::env::var("OBSERVABILITY_BACKEND").unwrap_or_else(|_| "none".into()), }, diff --git a/src/config/transcription.rs b/src/config/transcription.rs new file mode 100644 index 00000000..b0f76066 --- /dev/null +++ b/src/config/transcription.rs @@ -0,0 +1,79 @@ +use secrecy::SecretString; + +use crate::config::helpers::{optional_env, parse_bool_env}; +use crate::error::ConfigError; +use crate::settings::Settings; + +/// Transcription pipeline configuration. +#[derive(Debug, Clone)] +pub struct TranscriptionConfig { + /// Whether audio transcription is enabled. + pub enabled: bool, + /// Provider: "openai" (default). + pub provider: String, + /// OpenAI API key (reuses OPENAI_API_KEY). + pub openai_api_key: Option, + /// Model to use (default: "whisper-1"). + pub model: String, + /// Base URL override for the transcription API. + pub base_url: Option, +} + +impl Default for TranscriptionConfig { + fn default() -> Self { + Self { + enabled: false, + provider: "openai".to_string(), + openai_api_key: None, + model: "whisper-1".to_string(), + base_url: None, + } + } +} + +impl TranscriptionConfig { + pub(crate) fn resolve(settings: &Settings) -> Result { + let enabled = parse_bool_env( + "TRANSCRIPTION_ENABLED", + settings.transcription.as_ref().is_some_and(|t| t.enabled), + )?; + + let provider = + optional_env("TRANSCRIPTION_PROVIDER")?.unwrap_or_else(|| "openai".to_string()); + + let openai_api_key = optional_env("OPENAI_API_KEY")?.map(SecretString::from); + + let model = optional_env("TRANSCRIPTION_MODEL")?.unwrap_or_else(|| "whisper-1".to_string()); + + let base_url = optional_env("TRANSCRIPTION_BASE_URL")?; + + Ok(Self { + enabled, + provider, + openai_api_key, + model, + base_url, + }) + } + + /// Create the transcription provider if enabled and configured. + pub fn create_provider(&self) -> Option> { + if !self.enabled { + return None; + } + + // Currently only OpenAI Whisper is supported; more providers can be + // added here with a match on self.provider. + let api_key = self.openai_api_key.as_ref()?; + tracing::info!(model = %self.model, "Audio transcription enabled via OpenAI Whisper"); + + let mut provider = crate::transcription::OpenAiWhisperProvider::new(api_key.clone()) + .with_model(&self.model); + + if let Some(ref base_url) = self.base_url { + provider = provider.with_base_url(base_url); + } + + Some(Box::new(provider)) + } +} diff --git a/src/document_extraction/extractors.rs b/src/document_extraction/extractors.rs new file mode 100644 index 00000000..ddb30911 --- /dev/null +++ b/src/document_extraction/extractors.rs @@ -0,0 +1,514 @@ +//! Format-specific text extraction routines. + +use std::io::Read; + +/// Extract text from document bytes based on MIME type and optional filename. +pub fn extract_text(data: &[u8], mime: &str, filename: Option<&str>) -> Result { + let base_mime = mime.split(';').next().unwrap_or(mime).trim(); + + match base_mime { + // PDF + "application/pdf" => extract_pdf(data), + + // Office XML formats + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" => { + extract_docx(data) + } + "application/vnd.openxmlformats-officedocument.presentationml.presentation" => { + extract_pptx(data) + } + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => extract_xlsx(data), + + // Legacy Office (best-effort: treat as binary, try text extraction) + "application/msword" | "application/vnd.ms-powerpoint" | "application/vnd.ms-excel" => { + // Legacy binary formats — try to extract any text strings + extract_binary_strings(data) + } + + // Plain text family + "text/plain" + | "text/csv" + | "text/tab-separated-values" + | "text/markdown" + | "text/html" + | "text/xml" + | "text/x-python" + | "text/x-java" + | "text/x-c" + | "text/x-c++" + | "text/x-rust" + | "text/x-go" + | "text/x-ruby" + | "text/x-shellscript" + | "text/javascript" + | "text/css" + | "text/x-toml" + | "text/x-yaml" + | "text/x-log" => extract_utf8(data), + + // JSON / XML / YAML application types + "application/json" | "application/xml" | "application/x-yaml" | "application/yaml" + | "application/toml" | "application/x-sh" => extract_utf8(data), + + // RTF + "application/rtf" | "text/rtf" => extract_rtf(data), + + // Fallback: try to infer from filename extension + _ => { + if let Some(text) = try_extract_by_extension(data, filename) { + Ok(text) + } else { + Err(format!("unsupported document type: {base_mime}")) + } + } + } +} + +fn extract_pdf(data: &[u8]) -> Result { + pdf_extract::extract_text_from_mem(data) + .map(|t| t.trim().to_string()) + .map_err(|e| format!("PDF extraction failed: {e}")) +} + +fn extract_docx(data: &[u8]) -> Result { + extract_office_xml(data, "word/document.xml") +} + +fn extract_pptx(data: &[u8]) -> Result { + let cursor = std::io::Cursor::new(data); + let mut archive = + zip::ZipArchive::new(cursor).map_err(|e| format!("invalid PPTX archive: {e}"))?; + + // Collect slide filenames (ppt/slides/slide1.xml, slide2.xml, ...) + let mut slide_names: Vec = Vec::new(); + for i in 0..archive.len() { + if let Ok(file) = archive.by_index(i) { + let name = file.name().to_string(); + if name.starts_with("ppt/slides/slide") && name.ends_with(".xml") { + slide_names.push(name); + } + } + } + slide_names.sort(); + + let mut all_text = Vec::new(); + for name in &slide_names { + if let Ok(mut file) = archive.by_name(name) { + let mut xml = String::new(); + if file.read_to_string(&mut xml).is_ok() { + let text = strip_xml_tags(&xml); + if !text.is_empty() { + all_text.push(text); + } + } + } + } + + if all_text.is_empty() { + return Err("no text found in PPTX slides".to_string()); + } + Ok(all_text.join("\n\n---\n\n")) +} + +fn extract_xlsx(data: &[u8]) -> Result { + let cursor = std::io::Cursor::new(data); + let mut archive = + zip::ZipArchive::new(cursor).map_err(|e| format!("invalid XLSX archive: {e}"))?; + + // Read shared strings (xl/sharedStrings.xml) + let shared_strings = if let Ok(mut file) = archive.by_name("xl/sharedStrings.xml") { + let mut xml = String::new(); + file.read_to_string(&mut xml) + .map_err(|e| format!("failed to read shared strings: {e}"))?; + parse_xlsx_shared_strings(&xml) + } else { + Vec::new() + }; + + // Read sheet data + let mut sheet_names: Vec = Vec::new(); + for i in 0..archive.len() { + if let Ok(file) = archive.by_index(i) { + let name = file.name().to_string(); + if name.starts_with("xl/worksheets/sheet") && name.ends_with(".xml") { + sheet_names.push(name); + } + } + } + sheet_names.sort(); + + let mut all_text = Vec::new(); + for name in &sheet_names { + if let Ok(mut file) = archive.by_name(name) { + let mut xml = String::new(); + if file.read_to_string(&mut xml).is_ok() { + let text = parse_xlsx_sheet(&xml, &shared_strings); + if !text.is_empty() { + all_text.push(text); + } + } + } + } + + if all_text.is_empty() && !shared_strings.is_empty() { + // Fallback: just return shared strings + return Ok(shared_strings.join("\n")); + } + + if all_text.is_empty() { + return Err("no text found in XLSX".to_string()); + } + Ok(all_text.join("\n\n")) +} + +fn extract_office_xml(data: &[u8], content_path: &str) -> Result { + let cursor = std::io::Cursor::new(data); + let mut archive = + zip::ZipArchive::new(cursor).map_err(|e| format!("invalid Office XML archive: {e}"))?; + + let mut file = archive + .by_name(content_path) + .map_err(|e| format!("content file not found in archive: {e}"))?; + + let mut xml = String::new(); + file.read_to_string(&mut xml) + .map_err(|e| format!("failed to read content: {e}"))?; + + let text = strip_xml_tags(&xml); + if text.is_empty() { + return Err("no text content found".to_string()); + } + Ok(text) +} + +fn extract_utf8(data: &[u8]) -> Result { + // Try UTF-8 first, fall back to lossy decoding + match std::str::from_utf8(data) { + Ok(s) => Ok(s.to_string()), + Err(_) => Ok(String::from_utf8_lossy(data).to_string()), + } +} + +fn extract_rtf(data: &[u8]) -> Result { + // Basic RTF text extraction: strip control words and groups + let text = String::from_utf8_lossy(data); + let mut result = String::new(); + let mut depth = 0i32; + let mut chars = text.chars().peekable(); + + while let Some(ch) = chars.next() { + match ch { + '{' => depth += 1, + '}' => depth = (depth - 1).max(0), + '\\' => { + // Skip control word + let mut word = String::new(); + while let Some(&next) = chars.peek() { + if next.is_ascii_alphabetic() { + word.push(chars.next().unwrap()); + } else { + break; + } + } + // Skip optional numeric parameter + while let Some(&next) = chars.peek() { + if next.is_ascii_digit() || next == '-' { + chars.next(); + } else { + break; + } + } + // Consume trailing space + if let Some(&' ') = chars.peek() { + chars.next(); + } + // Convert common control words to text + match word.as_str() { + "par" | "line" => result.push('\n'), + "tab" => result.push('\t'), + _ => {} + } + } + _ => { + if depth <= 1 { + result.push(ch); + } + } + } + } + + let trimmed = result.trim().to_string(); + if trimmed.is_empty() { + return Err("no text found in RTF".to_string()); + } + Ok(trimmed) +} + +fn extract_binary_strings(data: &[u8]) -> Result { + // Extract printable ASCII/UTF-8 runs from binary data (last resort) + let mut strings = Vec::new(); + let mut current = String::new(); + + for &byte in data { + if (0x20..0x7F).contains(&byte) { + current.push(byte as char); + } else { + if current.len() >= 4 { + strings.push(std::mem::take(&mut current)); + } + current.clear(); + } + } + if current.len() >= 4 { + strings.push(current); + } + + if strings.is_empty() { + return Err("no readable text in binary document".to_string()); + } + Ok(strings.join(" ")) +} + +/// Strip XML tags and return just the text content. +fn strip_xml_tags(xml: &str) -> String { + let mut result = String::with_capacity(xml.len() / 2); + let mut in_tag = false; + let mut last_was_space = true; + + for ch in xml.chars() { + match ch { + '<' => { + in_tag = true; + } + '>' => { + in_tag = false; + // Add space between tag-delimited text runs + if !last_was_space && !result.is_empty() { + result.push(' '); + last_was_space = true; + } + } + _ if !in_tag => { + if ch.is_whitespace() { + if !last_was_space { + result.push(' '); + last_was_space = true; + } + } else { + result.push(ch); + last_was_space = false; + } + } + _ => {} + } + } + + // Decode common XML entities + result + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .trim() + .to_string() +} + +/// Parse XLSX shared strings XML into a Vec of strings. +fn parse_xlsx_shared_strings(xml: &str) -> Vec { + // Shared strings are in text elements + let mut strings = Vec::new(); + let mut in_t = false; + let mut current = String::new(); + let mut in_tag = false; + let mut tag_name = String::new(); + + for ch in xml.chars() { + match ch { + '<' => { + in_tag = true; + tag_name.clear(); + } + '>' => { + in_tag = false; + let tag = tag_name.trim().to_string(); + if tag == "t" || tag.starts_with("t ") { + in_t = true; + current.clear(); + } else if tag == "/t" { + in_t = false; + strings.push(std::mem::take(&mut current)); + } else if tag == "/si" { + in_t = false; + } + } + _ if in_tag => { + tag_name.push(ch); + } + _ if in_t => { + current.push(ch); + } + _ => {} + } + } + + strings +} + +/// Parse XLSX sheet XML into tab-separated rows. +fn parse_xlsx_sheet(xml: &str, shared_strings: &[String]) -> String { + // Simple extraction: find values in cells, resolve shared string refs + let mut rows: Vec> = Vec::new(); + let mut current_row: Vec = Vec::new(); + let mut in_v = false; + let mut in_row = false; + let mut current_val = String::new(); + let mut cell_type = String::new(); + let mut in_tag = false; + let mut tag_buf = String::new(); + + for ch in xml.chars() { + match ch { + '<' => { + in_tag = true; + tag_buf.clear(); + } + '>' => { + in_tag = false; + let tag = tag_buf.trim().to_string(); + if tag == "row" || tag.starts_with("row ") { + in_row = true; + current_row.clear(); + } else if tag == "/row" { + in_row = false; + if !current_row.is_empty() { + rows.push(std::mem::take(&mut current_row)); + } + } else if in_row && (tag.starts_with("c ") || tag == "c") { + // Extract type attribute: t="s" means shared string + cell_type.clear(); + if let Some(t_pos) = tag.find("t=\"") { + let rest = &tag[t_pos + 3..]; + if let Some(end) = rest.find('"') { + cell_type = rest[..end].to_string(); + } + } + } else if tag == "v" || tag.starts_with("v ") { + in_v = true; + current_val.clear(); + } else if tag == "/v" { + in_v = false; + let val = if cell_type == "s" { + // Shared string reference + current_val + .trim() + .parse::() + .ok() + .and_then(|idx| shared_strings.get(idx)) + .cloned() + .unwrap_or_default() + } else { + current_val.clone() + }; + current_row.push(val); + } else if tag == "/c" { + cell_type.clear(); + } + } + _ if in_tag => { + tag_buf.push(ch); + } + _ if in_v => { + current_val.push(ch); + } + _ => {} + } + } + + rows.iter() + .map(|row| row.join("\t")) + .collect::>() + .join("\n") +} + +/// Try to extract text based on filename extension when MIME type is generic. +fn try_extract_by_extension(data: &[u8], filename: Option<&str>) -> Option { + let ext = filename?.rsplit('.').next()?.to_lowercase(); + + match ext.as_str() { + "pdf" => extract_pdf(data).ok(), + "docx" => extract_docx(data).ok(), + "pptx" => extract_pptx(data).ok(), + "xlsx" => extract_xlsx(data).ok(), + "doc" | "ppt" | "xls" => extract_binary_strings(data).ok(), + "rtf" => extract_rtf(data).ok(), + "txt" | "csv" | "tsv" | "json" | "xml" | "yaml" | "yml" | "toml" | "md" | "markdown" + | "py" | "js" | "ts" | "rs" | "go" | "java" | "c" | "cpp" | "h" | "hpp" | "rb" | "sh" + | "bash" | "zsh" | "fish" | "css" | "html" | "htm" | "sql" | "log" | "ini" | "cfg" + | "conf" | "env" | "gitignore" | "dockerfile" => extract_utf8(data).ok(), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn strip_xml_basic() { + let xml = "

Hello

World

"; + assert_eq!(strip_xml_tags(xml), "Hello World"); + } + + #[test] + fn strip_xml_entities() { + let xml = "A & B < C"; + assert_eq!(strip_xml_tags(xml), "A & B < C"); + } + + #[test] + fn extract_utf8_valid() { + assert_eq!(extract_utf8(b"hello").unwrap(), "hello"); + } + + #[test] + fn extract_utf8_lossy() { + let data = b"hello \xff world"; + let result = extract_utf8(data).unwrap(); + assert!(result.contains("hello")); + assert!(result.contains("world")); + } + + #[test] + fn extract_by_extension_txt() { + let result = try_extract_by_extension(b"content", Some("notes.txt")); + assert_eq!(result, Some("content".to_string())); + } + + #[test] + fn extract_by_extension_unknown() { + let result = try_extract_by_extension(b"data", Some("file.xyz")); + assert!(result.is_none()); + } + + #[test] + fn extract_by_extension_no_filename() { + let result = try_extract_by_extension(b"data", None); + assert!(result.is_none()); + } + + #[test] + fn rtf_basic_extraction() { + let rtf = br"{\rtf1\ansi Hello World\par Second line}"; + let result = extract_rtf(rtf).unwrap(); + assert!(result.contains("Hello World")); + assert!(result.contains("Second line")); + } + + #[test] + fn xlsx_shared_strings_parsing() { + let xml = r#"NameAge"#; + let strings = parse_xlsx_shared_strings(xml); + assert_eq!(strings, vec!["Name", "Age"]); + } +} diff --git a/src/document_extraction/mod.rs b/src/document_extraction/mod.rs new file mode 100644 index 00000000..9376c17c --- /dev/null +++ b/src/document_extraction/mod.rs @@ -0,0 +1,283 @@ +//! Document text extraction pipeline. +//! +//! Provides a [`DocumentExtractionMiddleware`] that detects document attachments +//! on incoming messages and extracts text content so the LLM can reason about them. +//! +//! Supported formats: +//! - **PDF** — via `pdf-extract` +//! - **Office XML** (DOCX, PPTX, XLSX) — ZIP + XML text extraction +//! - **Plain text** (TXT, CSV, JSON, XML, Markdown, code) — UTF-8 decode + +mod extractors; + +use crate::channels::{AttachmentKind, IncomingMessage}; + +/// Maximum document size to extract (10 MB). +const MAX_DOCUMENT_SIZE: u64 = 10 * 1024 * 1024; + +/// Maximum extracted text length to keep (100K chars ≈ ~25K tokens). +const MAX_EXTRACTED_TEXT_LEN: usize = 100_000; + +/// Middleware that processes document attachments on incoming messages. +/// +/// For each document attachment with inline data, attempts to: +/// 1. Extract text based on MIME type +/// 2. Set `extracted_text` on the attachment +/// +/// Downloading from `source_url` is intentionally not supported to prevent SSRF. +/// Channels must populate `attachment.data` via `store_attachment_data`. +#[derive(Default)] +pub struct DocumentExtractionMiddleware; + +impl DocumentExtractionMiddleware { + pub fn new() -> Self { + Self + } + + /// Process an incoming message, extracting text from document attachments. + pub async fn process(&self, msg: &mut IncomingMessage) { + let mut extractions = Vec::new(); + + for (i, attachment) in msg.attachments.iter().enumerate() { + if attachment.kind != AttachmentKind::Document { + continue; + } + if attachment.extracted_text.is_some() { + continue; + } + + // Check if too large + if let Some(size) = attachment.size_bytes.filter(|&s| s > MAX_DOCUMENT_SIZE) { + tracing::warn!( + attachment_id = %attachment.id, + size, + "Document too large for extraction, skipping" + ); + let mb = size as f64 / (1024.0 * 1024.0); + let max_mb = MAX_DOCUMENT_SIZE as f64 / (1024.0 * 1024.0); + extractions.push(( + i, + format!( + "[Document too large for text extraction: {mb:.1} MB exceeds {max_mb:.0} MB limit. \ + Please send a smaller file or copy-paste the relevant text.]" + ), + )); + continue; + } + + // Use inline data only — downloading from source_url is intentionally + // not supported to prevent SSRF. Channels must populate attachment.data + // via store_attachment_data before emitting the message. + if attachment.data.is_empty() { + extractions.push(( + i, + "[Document has no inline data. \ + Please try sending the file again.]" + .to_string(), + )); + continue; + } + + // Enforce size limit before cloning to avoid unnecessary allocation + if attachment.data.len() as u64 > MAX_DOCUMENT_SIZE { + let mb = attachment.data.len() as f64 / (1024.0 * 1024.0); + let max_mb = MAX_DOCUMENT_SIZE as f64 / (1024.0 * 1024.0); + extractions.push(( + i, + format!( + "[Document too large for text extraction: {mb:.1} MB exceeds {max_mb:.0} MB limit. \ + Please send a smaller file or copy-paste the relevant text.]" + ), + )); + continue; + } + + let data = attachment.data.clone(); + + let mime = &attachment.mime_type; + let filename = attachment.filename.as_deref(); + match extractors::extract_text(&data, mime, filename) { + Ok(text) => { + // Truncate at a char boundary to avoid panicking on multi-byte UTF-8 + let text = if text.len() > MAX_EXTRACTED_TEXT_LEN { + let boundary = text + .char_indices() + .map(|(i, _)| i) + .take_while(|&i| i <= MAX_EXTRACTED_TEXT_LEN) + .last() + .unwrap_or(0); + let mut truncated = text[..boundary].to_string(); + truncated.push_str("\n\n[... truncated, document too long ...]"); + truncated + } else { + text + }; + tracing::info!( + attachment_id = %attachment.id, + mime_type = %mime, + text_len = text.len(), + "Extracted text from document" + ); + extractions.push((i, text)); + } + Err(e) => { + tracing::warn!( + attachment_id = %attachment.id, + mime_type = %mime, + error = %e, + "Failed to extract text from document" + ); + let name = filename.unwrap_or("document"); + extractions.push(( + i, + format!( + "[Failed to extract text from '{name}' ({mime}): {e}. \ + The file format may not be supported.]" + ), + )); + } + } + } + + for (i, text) in extractions { + msg.attachments[i].extracted_text = Some(text); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::channels::IncomingAttachment; + + fn doc_attachment(mime: &str, filename: &str, data: Vec) -> IncomingAttachment { + IncomingAttachment { + id: "doc_1".to_string(), + kind: AttachmentKind::Document, + mime_type: mime.to_string(), + filename: Some(filename.to_string()), + size_bytes: Some(data.len() as u64), + source_url: None, + storage_key: None, + extracted_text: None, + data, + duration_secs: None, + } + } + + #[tokio::test] + async fn extracts_plain_text() { + let middleware = DocumentExtractionMiddleware::new(); + let mut msg = IncomingMessage::new("test", "user1", "check this").with_attachments(vec![ + doc_attachment("text/plain", "notes.txt", b"Hello world".to_vec()), + ]); + + middleware.process(&mut msg).await; + assert_eq!( + msg.attachments[0].extracted_text.as_deref(), + Some("Hello world") + ); + } + + #[tokio::test] + async fn extracts_csv() { + let middleware = DocumentExtractionMiddleware::new(); + let mut msg = IncomingMessage::new("test", "user1", "analyze").with_attachments(vec![ + doc_attachment("text/csv", "data.csv", b"name,age\nAlice,30".to_vec()), + ]); + + middleware.process(&mut msg).await; + assert_eq!( + msg.attachments[0].extracted_text.as_deref(), + Some("name,age\nAlice,30") + ); + } + + #[tokio::test] + async fn extracts_json() { + let middleware = DocumentExtractionMiddleware::new(); + let data = br#"{"key": "value"}"#.to_vec(); + let mut msg = IncomingMessage::new("test", "user1", "parse") + .with_attachments(vec![doc_attachment("application/json", "data.json", data)]); + + middleware.process(&mut msg).await; + assert!(msg.attachments[0].extracted_text.is_some()); + } + + #[tokio::test] + async fn skips_already_extracted() { + let middleware = DocumentExtractionMiddleware::new(); + let mut att = doc_attachment("text/plain", "test.txt", b"data".to_vec()); + att.extracted_text = Some("Already done".to_string()); + let mut msg = IncomingMessage::new("test", "user1", "").with_attachments(vec![att]); + + middleware.process(&mut msg).await; + assert_eq!( + msg.attachments[0].extracted_text.as_deref(), + Some("Already done") + ); + } + + #[tokio::test] + async fn skips_audio_attachments() { + let middleware = DocumentExtractionMiddleware::new(); + let mut att = doc_attachment("text/plain", "test.txt", b"data".to_vec()); + att.kind = AttachmentKind::Audio; + let mut msg = IncomingMessage::new("test", "user1", "").with_attachments(vec![att]); + + middleware.process(&mut msg).await; + assert!(msg.attachments[0].extracted_text.is_none()); + } + + #[tokio::test] + async fn reports_oversized_documents() { + let middleware = DocumentExtractionMiddleware::new(); + let mut att = doc_attachment("text/plain", "huge.txt", vec![]); + att.size_bytes = Some(MAX_DOCUMENT_SIZE + 1); + let mut msg = IncomingMessage::new("test", "user1", "").with_attachments(vec![att]); + + middleware.process(&mut msg).await; + let text = msg.attachments[0].extracted_text.as_deref().unwrap(); + assert!( + text.contains("too large"), + "Expected 'too large' error, got: {text}" + ); + } + + #[tokio::test] + async fn truncates_long_text() { + let middleware = DocumentExtractionMiddleware::new(); + let long_text = "x".repeat(MAX_EXTRACTED_TEXT_LEN + 1000); + let mut msg = + IncomingMessage::new("test", "user1", "read").with_attachments(vec![doc_attachment( + "text/plain", + "long.txt", + long_text.into_bytes(), + )]); + + middleware.process(&mut msg).await; + let extracted = msg.attachments[0].extracted_text.as_ref().unwrap(); + assert!(extracted.len() < MAX_EXTRACTED_TEXT_LEN + 100); + assert!(extracted.ends_with("[... truncated, document too long ...]")); + } + + #[tokio::test] + async fn extracts_pdf_text() { + // Minimal valid PDF with text "Hello World" + let pdf_bytes = include_bytes!("../../tests/fixtures/hello.pdf"); + let middleware = DocumentExtractionMiddleware::new(); + let mut msg = + IncomingMessage::new("test", "user1", "review").with_attachments(vec![doc_attachment( + "application/pdf", + "hello.pdf", + pdf_bytes.to_vec(), + )]); + + middleware.process(&mut msg).await; + let text = msg.attachments[0].extracted_text.as_deref().unwrap_or(""); + assert!( + text.contains("Hello"), + "PDF extraction should contain 'Hello', got: {text}" + ); + } +} diff --git a/src/extensions/discovery.rs b/src/extensions/discovery.rs index a9c625d7..b58101bc 100644 --- a/src/extensions/discovery.rs +++ b/src/extensions/discovery.rs @@ -106,6 +106,7 @@ impl OnlineDiscovery { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }) } else { None @@ -181,6 +182,7 @@ impl OnlineDiscovery { source: ExtensionSource::Discovered { url }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }) }) .collect() diff --git a/src/extensions/manager.rs b/src/extensions/manager.rs index 664a9d16..6ef47d22 100644 --- a/src/extensions/manager.rs +++ b/src/extensions/manager.rs @@ -18,7 +18,8 @@ use crate::extensions::discovery::OnlineDiscovery; use crate::extensions::registry::ExtensionRegistry; use crate::extensions::{ ActivateResult, AuthResult, ExtensionError, ExtensionKind, ExtensionSource, InstallResult, - InstalledExtension, RegistryEntry, ResultSource, SearchResult, ToolAuthState, + InstalledExtension, RegistryEntry, ResultSource, SearchResult, ToolAuthState, UpgradeOutcome, + UpgradeResult, }; use crate::hooks::HookRegistry; use crate::pairing::PairingStore; @@ -412,6 +413,7 @@ impl ExtensionManager { has_auth: false, installed: true, activation_error: None, + version: None, }); } } @@ -427,15 +429,28 @@ impl ExtensionManager { { match discover_tools(&self.wasm_tools_dir).await { Ok(tools) => { - for (name, _discovered) in tools { + for (name, discovered) in tools { let active = self.tool_registry.has(&name).await; - let display_name = self + let registry_entry = self .registry .get_with_kind(&name, Some(ExtensionKind::WasmTool)) - .await - .map(|e| e.display_name); + .await; + let display_name = registry_entry.as_ref().map(|e| e.display_name.clone()); let auth_state = self.check_tool_auth_status(&name).await; + let version = if let Some(ref cap_path) = discovered.capabilities_path { + tokio::fs::read(cap_path) + .await + .ok() + .and_then(|bytes| { + crate::tools::wasm::CapabilitiesFile::from_bytes(&bytes).ok() + }) + .and_then(|cap| cap.version) + } else { + None + }; + let version = + version.or_else(|| registry_entry.and_then(|e| e.version.clone())); extensions.push(InstalledExtension { name: name.clone(), kind: ExtensionKind::WasmTool, @@ -449,6 +464,7 @@ impl ExtensionManager { has_auth: auth_state != ToolAuthState::NoAuth, installed: true, activation_error: None, + version, }); } } @@ -466,15 +482,31 @@ impl ExtensionManager { Ok(channels) => { let active_names = self.active_channel_names.read().await; let errors = self.activation_errors.read().await; - for (name, _discovered) in channels { + for (name, discovered) in channels { let active = active_names.contains(&name); let auth_state = self.check_channel_auth_status(&name).await; let activation_error = errors.get(&name).cloned(); - let display_name = self + let registry_entry = self .registry .get_with_kind(&name, Some(ExtensionKind::WasmChannel)) - .await - .map(|e| e.display_name); + .await; + let display_name = registry_entry.as_ref().map(|e| e.display_name.clone()); + let version = if let Some(ref cap_path) = discovered.capabilities_path { + tokio::fs::read(cap_path) + .await + .ok() + .and_then(|bytes| { + crate::channels::wasm::ChannelCapabilitiesFile::from_bytes( + &bytes, + ) + .ok() + }) + .and_then(|cap| cap.version) + } else { + None + }; + let version = + version.or_else(|| registry_entry.and_then(|e| e.version.clone())); extensions.push(InstalledExtension { name, kind: ExtensionKind::WasmChannel, @@ -488,6 +520,7 @@ impl ExtensionManager { has_auth: false, installed: true, activation_error, + version, }); } } @@ -526,6 +559,7 @@ impl ExtensionManager { has_auth: false, installed: false, activation_error: None, + version: entry.version, }); } } @@ -637,6 +671,207 @@ impl ExtensionManager { } } + /// Upgrade installed WASM extensions to match the current host WIT version. + /// + /// If `name` is `Some`, upgrades only that extension. If `None`, checks all + /// installed WASM tools and channels and upgrades any that are outdated. + /// + /// The upgrade preserves authentication secrets — only the `.wasm` binary + /// (and `.capabilities.json`) are replaced. + pub async fn upgrade(&self, name: Option<&str>) -> Result { + // Collect extensions to check + let mut candidates: Vec<(String, ExtensionKind)> = Vec::new(); + + if let Some(name) = name { + Self::validate_extension_name(name)?; + let kind = self.determine_installed_kind(name).await?; + if kind == ExtensionKind::McpServer { + return Err(ExtensionError::Other( + "MCP servers don't have WIT versions and cannot be upgraded this way" + .to_string(), + )); + } + candidates.push((name.to_string(), kind)); + } else { + // Discover all installed WASM tools + if self.wasm_tools_dir.exists() + && let Ok(tools) = discover_tools(&self.wasm_tools_dir).await + { + for (tool_name, _) in tools { + candidates.push((tool_name, ExtensionKind::WasmTool)); + } + } + // Discover all installed WASM channels + if self.wasm_channels_dir.exists() + && let Ok(channels) = + crate::channels::wasm::discover_channels(&self.wasm_channels_dir).await + { + for (ch_name, _) in channels { + candidates.push((ch_name, ExtensionKind::WasmChannel)); + } + } + } + + if candidates.is_empty() { + return Ok(UpgradeResult { + results: Vec::new(), + message: "No WASM extensions installed.".to_string(), + }); + } + + let mut outcomes = Vec::new(); + + for (ext_name, kind) in &candidates { + let outcome = self.upgrade_one(ext_name, *kind).await; + outcomes.push(outcome); + } + + let upgraded = outcomes.iter().filter(|o| o.status == "upgraded").count(); + let up_to_date = outcomes + .iter() + .filter(|o| o.status == "already_up_to_date") + .count(); + let failed = outcomes.iter().filter(|o| o.status == "failed").count(); + + let message = format!( + "{} extension(s) checked: {} upgraded, {} already up to date, {} failed", + outcomes.len(), + upgraded, + up_to_date, + failed + ); + + Ok(UpgradeResult { + results: outcomes, + message, + }) + } + + /// Upgrade a single WASM extension if its WIT version is outdated. + async fn upgrade_one(&self, name: &str, kind: ExtensionKind) -> UpgradeOutcome { + let (cap_dir, host_wit) = match kind { + ExtensionKind::WasmTool => (&self.wasm_tools_dir, crate::tools::wasm::WIT_TOOL_VERSION), + ExtensionKind::WasmChannel => ( + &self.wasm_channels_dir, + crate::tools::wasm::WIT_CHANNEL_VERSION, + ), + ExtensionKind::McpServer => { + return UpgradeOutcome { + name: name.to_string(), + kind, + status: "failed".to_string(), + detail: "MCP servers cannot be upgraded this way".to_string(), + }; + } + }; + + // Read current WIT version from capabilities + let cap_path = cap_dir.join(format!("{}.capabilities.json", name)); + let declared_wit = if cap_path.exists() { + match tokio::fs::read(&cap_path).await { + Ok(bytes) => { + let wit: Option = match kind { + ExtensionKind::WasmTool => { + crate::tools::wasm::CapabilitiesFile::from_bytes(&bytes) + .ok() + .and_then(|c| c.wit_version) + } + ExtensionKind::WasmChannel => { + crate::channels::wasm::ChannelCapabilitiesFile::from_bytes(&bytes) + .ok() + .and_then(|c| c.wit_version) + } + ExtensionKind::McpServer => None, + }; + wit + } + Err(_) => None, + } + } else { + None + }; + + // Check if upgrade is needed + let needs_upgrade = + crate::tools::wasm::check_wit_version_compat(name, declared_wit.as_deref(), host_wit) + .is_err(); + + if !needs_upgrade { + return UpgradeOutcome { + name: name.to_string(), + kind, + status: "already_up_to_date".to_string(), + detail: format!( + "WIT {} matches host WIT {}", + declared_wit.as_deref().unwrap_or("unknown"), + host_wit + ), + }; + } + + // Check registry for a newer version + let entry = self.registry.get_with_kind(name, Some(kind)).await; + let Some(entry) = entry else { + return UpgradeOutcome { + name: name.to_string(), + kind, + status: "not_in_registry".to_string(), + detail: format!( + "Extension '{}' has outdated WIT {} (host: {}), \ + but is not in the registry. Reinstall manually with a URL.", + name, + declared_wit.as_deref().unwrap_or("unknown"), + host_wit + ), + }; + }; + + // Delete old .wasm file (keep secrets intact) + let wasm_path = cap_dir.join(format!("{}.wasm", name)); + if wasm_path.exists() + && let Err(e) = tokio::fs::remove_file(&wasm_path).await + { + return UpgradeOutcome { + name: name.to_string(), + kind, + status: "failed".to_string(), + detail: format!("Failed to remove old WASM binary: {}", e), + }; + } + // Also remove old capabilities so install_from_entry can write the new one + if cap_path.exists() { + let _ = tokio::fs::remove_file(&cap_path).await; + } + + // Reinstall from registry + match self.install_from_entry(&entry).await { + Ok(_) => { + tracing::info!( + extension = %name, + old_wit = ?declared_wit, + new_host_wit = %host_wit, + "Upgraded WASM extension" + ); + UpgradeOutcome { + name: name.to_string(), + kind, + status: "upgraded".to_string(), + detail: format!( + "Upgraded from WIT {} to host WIT {}. Restart to activate.", + declared_wit.as_deref().unwrap_or("unknown"), + host_wit + ), + } + } + Err(e) => UpgradeOutcome { + name: name.to_string(), + kind, + status: "failed".to_string(), + detail: format!("Reinstall failed: {}. Old files were removed.", e), + }, + } + } + /// Get detailed info about an installed extension (version, wit_version, host compatibility). pub async fn extension_info(&self, name: &str) -> Result { Self::validate_extension_name(name)?; @@ -3336,6 +3571,7 @@ fn combine_install_errors( mod tests { use std::sync::Arc; + use crate::extensions::ExtensionManager; use crate::extensions::manager::{ FallbackDecision, combine_install_errors, fallback_decision, infer_kind_from_url, }; @@ -3621,4 +3857,107 @@ mod tests { assert_eq!(std::fs::read_to_string(&tool_cap).unwrap(), tool_caps); assert_eq!(std::fs::read_to_string(&channel_cap).unwrap(), channel_caps); } + + #[tokio::test] + async fn test_upgrade_no_installed_extensions() { + let manager = make_manager_with_temp_dirs(); + let result = manager.upgrade(None).await.unwrap(); + assert!(result.results.is_empty()); + assert!(result.message.contains("No WASM extensions installed")); + } + + #[tokio::test] + async fn test_upgrade_mcp_server_rejected() { + let manager = make_manager_with_temp_dirs(); + // MCP servers can't be upgraded via tool_upgrade + let err = manager.upgrade(Some("some-mcp")).await; + // It will fail with NotInstalled because there's no MCP server named "some-mcp", + // but if it were installed, the MCP code path would be rejected. + assert!(err.is_err()); + } + + #[tokio::test] + async fn test_upgrade_up_to_date_extension() { + let dir = tempfile::tempdir().expect("temp dir"); + let channels_dir = dir.path().join("channels"); + std::fs::create_dir_all(&channels_dir).unwrap(); + + // Write a fake .wasm file and capabilities with current WIT version + let wasm_path = channels_dir.join("test-channel.wasm"); + std::fs::write(&wasm_path, b"\0asm fake").unwrap(); + + let cap_path = channels_dir.join("test-channel.capabilities.json"); + let caps = serde_json::json!({ + "type": "channel", + "name": "test-channel", + "wit_version": crate::tools::wasm::WIT_CHANNEL_VERSION, + }); + std::fs::write(&cap_path, serde_json::to_string(&caps).unwrap()).unwrap(); + + let manager = make_manager_custom_dirs(dir.path().join("tools"), channels_dir); + + let result = manager.upgrade(Some("test-channel")).await.unwrap(); + assert_eq!(result.results.len(), 1); + assert_eq!(result.results[0].status, "already_up_to_date"); + } + + #[tokio::test] + async fn test_upgrade_outdated_not_in_registry() { + let dir = tempfile::tempdir().expect("temp dir"); + let channels_dir = dir.path().join("channels"); + std::fs::create_dir_all(&channels_dir).unwrap(); + + // Write a fake .wasm file and capabilities with OLD WIT version + let wasm_path = channels_dir.join("custom-channel.wasm"); + std::fs::write(&wasm_path, b"\0asm fake").unwrap(); + + let cap_path = channels_dir.join("custom-channel.capabilities.json"); + let caps = serde_json::json!({ + "type": "channel", + "name": "custom-channel", + "wit_version": "0.1.0", + }); + std::fs::write(&cap_path, serde_json::to_string(&caps).unwrap()).unwrap(); + + let manager = make_manager_custom_dirs(dir.path().join("tools"), channels_dir); + + let result = manager.upgrade(Some("custom-channel")).await.unwrap(); + assert_eq!(result.results.len(), 1); + assert_eq!(result.results[0].status, "not_in_registry"); + } + + fn make_manager_with_temp_dirs() -> ExtensionManager { + let dir = tempfile::tempdir().expect("temp dir"); + make_manager_custom_dirs(dir.path().join("tools"), dir.path().join("channels")) + } + + fn make_manager_custom_dirs( + tools_dir: std::path::PathBuf, + channels_dir: std::path::PathBuf, + ) -> ExtensionManager { + use crate::secrets::{InMemorySecretsStore, SecretsCrypto}; + use crate::tools::ToolRegistry; + use crate::tools::mcp::session::McpSessionManager; + + std::fs::create_dir_all(&tools_dir).ok(); + std::fs::create_dir_all(&channels_dir).ok(); + + let master_key = + secrecy::SecretString::from("0123456789abcdef0123456789abcdef".to_string()); + let crypto = Arc::new(SecretsCrypto::new(master_key).unwrap()); + + ExtensionManager::new( + Arc::new(McpSessionManager::new()), + Arc::new(InMemorySecretsStore::new(crypto)), + Arc::new(ToolRegistry::new()), + None, + None, + tools_dir, + channels_dir, + None, + "test".to_string(), + None, + Vec::new(), + ) + } } diff --git a/src/extensions/mod.rs b/src/extensions/mod.rs index 1f0375e4..011d9571 100644 --- a/src/extensions/mod.rs +++ b/src/extensions/mod.rs @@ -70,6 +70,9 @@ pub struct RegistryEntry { pub fallback_source: Option>, /// How authentication works. pub auth_hint: AuthHint, + /// Extension version (semver), if known. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, } /// Where the extension binary or server lives. @@ -146,6 +149,26 @@ pub struct InstallResult { pub message: String, } +/// Result of upgrading one or more extensions. +#[derive(Debug, Clone, serde::Serialize)] +pub struct UpgradeResult { + /// Per-extension upgrade outcomes. + pub results: Vec, + /// Summary message. + pub message: String, +} + +/// Outcome for a single extension upgrade. +#[derive(Debug, Clone, serde::Serialize)] +pub struct UpgradeOutcome { + pub name: String, + pub kind: ExtensionKind, + /// What happened: "upgraded", "already_up_to_date", "failed", "not_in_registry". + pub status: String, + /// Human-readable detail. + pub detail: String, +} + /// Auth readiness state for the extensions list UI. /// /// Used by `check_tool_auth_status` and `check_channel_auth_status` to @@ -453,6 +476,9 @@ pub struct InstalledExtension { /// Last activation error for WASM channels. #[serde(skip_serializing_if = "Option::is_none")] pub activation_error: Option, + /// Extension version from capabilities file (semver). + #[serde(skip_serializing_if = "Option::is_none")] + pub version: Option, } /// Error type for extension operations. @@ -769,6 +795,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }; let sr = SearchResult { entry, @@ -798,6 +825,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::None, + version: None, }; let sr = SearchResult { entry, @@ -885,6 +913,7 @@ mod tests { has_auth: true, installed: false, activation_error: Some("token expired".to_string()), + version: None, }; let json = serde_json::to_value(&ext).unwrap(); assert_eq!(json["display_name"], "Gmail Tool"); diff --git a/src/extensions/registry.rs b/src/extensions/registry.rs index 14fa63bc..32dd4c2b 100644 --- a/src/extensions/registry.rs +++ b/src/extensions/registry.rs @@ -245,6 +245,7 @@ fn builtin_entries() -> Vec { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }, RegistryEntry { name: "linear".to_string(), @@ -265,6 +266,7 @@ fn builtin_entries() -> Vec { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }, RegistryEntry { name: "github".to_string(), @@ -285,6 +287,7 @@ fn builtin_entries() -> Vec { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }, RegistryEntry { name: "slack-mcp".to_string(), @@ -305,6 +308,7 @@ fn builtin_entries() -> Vec { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }, RegistryEntry { name: "sentry".to_string(), @@ -325,6 +329,7 @@ fn builtin_entries() -> Vec { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }, RegistryEntry { name: "stripe".to_string(), @@ -345,6 +350,7 @@ fn builtin_entries() -> Vec { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }, RegistryEntry { name: "cloudflare".to_string(), @@ -365,6 +371,7 @@ fn builtin_entries() -> Vec { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }, RegistryEntry { name: "asana".to_string(), @@ -383,6 +390,7 @@ fn builtin_entries() -> Vec { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }, RegistryEntry { name: "intercom".to_string(), @@ -402,6 +410,7 @@ fn builtin_entries() -> Vec { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }, // WASM channels (telegram, slack, discord, whatsapp) come from the embedded // registry catalog (registry/channels/*.json) with WasmDownload URLs pointing @@ -427,6 +436,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }; let score = score_entry(&entry, &["notion".to_string()]); @@ -450,6 +460,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }; let score = score_entry(&entry, &["calendar".to_string()]); @@ -473,6 +484,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }; let score = score_entry(&entry, &["wiki".to_string()]); @@ -496,6 +508,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }; let score = score_entry(&entry, &["xyzfoobar".to_string()]); @@ -560,6 +573,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }; registry.cache_discovered(vec![discovered]).await; @@ -586,6 +600,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::None, + version: None, }; registry.cache_discovered(vec![entry.clone()]).await; @@ -611,6 +626,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, + version: None, }, // This shares a name with the builtin slack-mcp but has a different kind, so both should appear RegistryEntry { @@ -626,6 +642,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, + version: None, }, ]; @@ -662,6 +679,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::Dcr, + version: None, }]; let registry = ExtensionRegistry::new_with_catalog(catalog_entries); @@ -689,6 +707,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, + version: None, }, RegistryEntry { name: "telegram".to_string(), @@ -703,6 +722,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, + version: None, }, ]; @@ -765,6 +785,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::None, + version: None, }; let channel_entry = RegistryEntry { name: "cached-ext".to_string(), @@ -779,6 +800,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::None, + version: None, }; registry @@ -822,6 +844,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, + version: None, }, RegistryEntry { name: "telegram".to_string(), @@ -836,6 +859,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::CapabilitiesAuth, + version: None, }, ]; @@ -884,6 +908,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::None, + version: None, }, RegistryEntry { name: "myext".to_string(), @@ -898,6 +923,7 @@ mod tests { }, fallback_source: None, auth_hint: AuthHint::None, + version: None, }, ]; diff --git a/src/lib.rs b/src/lib.rs index d14d14d2..fff5c5fa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -47,6 +47,7 @@ pub mod cli; pub mod config; pub mod context; pub mod db; +pub mod document_extraction; pub mod error; pub mod estimation; pub mod evaluation; @@ -67,6 +68,7 @@ pub mod setup; pub mod skills; pub mod tools; pub mod tracing_fmt; +pub mod transcription; pub mod tunnel; pub mod util; pub mod worker; diff --git a/src/llm/mod.rs b/src/llm/mod.rs index 54b77096..136ea240 100644 --- a/src/llm/mod.rs +++ b/src/llm/mod.rs @@ -25,8 +25,9 @@ pub use circuit_breaker::{CircuitBreakerConfig, CircuitBreakerProvider}; pub use failover::{CooldownConfig, FailoverProvider}; pub use nearai_chat::{ModelInfo, NearAiChatProvider}; pub use provider::{ - ChatMessage, CompletionRequest, CompletionResponse, FinishReason, LlmProvider, ModelMetadata, - Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, ToolDefinition, ToolResult, + ChatMessage, CompletionRequest, CompletionResponse, ContentPart, FinishReason, ImageUrl, + LlmProvider, ModelMetadata, Role, ToolCall, ToolCompletionRequest, ToolCompletionResponse, + ToolDefinition, ToolResult, }; pub use reasoning::{ ActionPlan, Reasoning, ReasoningContext, RespondOutput, RespondResult, SILENT_REPLY_TOKEN, diff --git a/src/llm/nearai_chat.rs b/src/llm/nearai_chat.rs index 6397d54c..7637cc08 100644 --- a/src/llm/nearai_chat.rs +++ b/src/llm/nearai_chat.rs @@ -671,11 +671,68 @@ struct ChatCompletionRequest { tool_choice: Option, } +/// Content field that serializes as either a string or an array of content parts. +/// +/// - `Text("hello")` → `"content": "hello"` +/// - `Parts([...])` → `"content": [{"type": "text", ...}, {"type": "image_url", ...}]` +#[derive(Debug, Clone)] +enum MessageContent { + Text(String), + Parts(Vec), +} + +impl Serialize for MessageContent { + fn serialize(&self, serializer: S) -> Result { + match self { + MessageContent::Text(s) => serializer.serialize_str(s), + MessageContent::Parts(parts) => parts.serialize(serializer), + } + } +} + +impl<'de> Deserialize<'de> for MessageContent { + fn deserialize>(deserializer: D) -> Result { + use serde::de; + use serde_json::Value; + + let val = Value::deserialize(deserializer)?; + match val { + Value::String(s) => Ok(MessageContent::Text(s)), + Value::Array(arr) => Ok(MessageContent::Text( + // For deserialization (responses), we only need the text content + arr.iter() + .find_map(|v| { + if v.get("type")?.as_str()? == "text" { + v.get("text")?.as_str().map(String::from) + } else { + None + } + }) + .unwrap_or_default(), + )), + Value::Null => Ok(MessageContent::Text(String::new())), + _ => Err(de::Error::custom( + "expected string, array, or null for content", + )), + } + } +} + +impl MessageContent { + fn as_text(&self) -> Option<&str> { + match self { + MessageContent::Text(s) if !s.is_empty() => Some(s), + MessageContent::Text(_) => None, + MessageContent::Parts(_) => None, + } + } +} + #[derive(Debug, Serialize, Deserialize)] struct ChatCompletionMessage { role: String, #[serde(skip_serializing_if = "Option::is_none")] - content: Option, + content: Option, #[serde(skip_serializing_if = "Option::is_none")] tool_call_id: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -843,10 +900,8 @@ fn flatten_tool_messages(messages: Vec) -> Vec = Vec::new(); - if let Some(ref text) = msg.content - && !text.is_empty() - { - parts.push(text.clone()); + if let Some(text) = msg.content.as_ref().and_then(|c| c.as_text()) { + parts.push(text.to_string()); } for tc in calls { parts.push(format!( @@ -856,7 +911,7 @@ fn flatten_tool_messages(messages: Vec) -> Vec) -> Vec for ChatCompletionMessage { let content = if role == "assistant" && tool_calls.is_some() && msg.content.is_empty() { None + } else if !msg.content_parts.is_empty() { + // Build multimodal content array: text + image parts + let mut parts = vec![crate::llm::ContentPart::Text { text: msg.content }]; + parts.extend(msg.content_parts); + Some(MessageContent::Parts(parts)) } else { - Some(msg.content) + Some(MessageContent::Text(msg.content)) }; Self { @@ -1072,7 +1135,10 @@ mod tests { let msg = ChatMessage::user("Hello"); let chat_msg: ChatCompletionMessage = msg.into(); assert_eq!(chat_msg.role, "user"); - assert_eq!(chat_msg.content, Some("Hello".to_string())); + assert_eq!( + chat_msg.content.as_ref().and_then(|c| c.as_text()), + Some("Hello") + ); } #[test] @@ -1146,14 +1212,14 @@ mod tests { let messages = vec![ ChatCompletionMessage { role: "system".to_string(), - content: Some("You are helpful.".to_string()), + content: Some(MessageContent::Text("You are helpful.".to_string())), tool_call_id: None, name: None, tool_calls: None, }, ChatCompletionMessage { role: "user".to_string(), - content: Some("Hello".to_string()), + content: Some(MessageContent::Text("Hello".to_string())), tool_call_id: None, name: None, tool_calls: None, @@ -1170,7 +1236,7 @@ mod tests { let messages = vec![ ChatCompletionMessage { role: "user".to_string(), - content: Some("test".to_string()), + content: Some(MessageContent::Text("test".to_string())), tool_call_id: None, name: None, tool_calls: None, @@ -1191,7 +1257,7 @@ mod tests { }, ChatCompletionMessage { role: "tool".to_string(), - content: Some("hi".to_string()), + content: Some(MessageContent::Text("hi".to_string())), tool_call_id: Some("call_1".to_string()), name: Some("echo".to_string()), tool_calls: None, @@ -1208,6 +1274,7 @@ mod tests { result[1] .content .as_ref() + .and_then(|c| c.as_text()) .unwrap() .contains("[Called tool `echo`") ); @@ -1219,6 +1286,7 @@ mod tests { result[2] .content .as_ref() + .and_then(|c| c.as_text()) .unwrap() .contains("[Tool `echo` returned: hi]") ); @@ -1229,7 +1297,7 @@ mod tests { let messages = vec![ ChatCompletionMessage { role: "assistant".to_string(), - content: Some("Let me check that.".to_string()), + content: Some(MessageContent::Text("Let me check that.".to_string())), tool_call_id: None, name: None, tool_calls: Some(vec![ChatCompletionToolCall { @@ -1243,7 +1311,7 @@ mod tests { }, ChatCompletionMessage { role: "tool".to_string(), - content: Some("found it".to_string()), + content: Some(MessageContent::Text("found it".to_string())), tool_call_id: Some("call_1".to_string()), name: Some("search".to_string()), tool_calls: None, @@ -1251,7 +1319,11 @@ mod tests { ]; let result = flatten_tool_messages(messages); - let text = result[0].content.as_ref().unwrap(); + let text = result[0] + .content + .as_ref() + .and_then(|c| c.as_text()) + .unwrap(); assert!(text.starts_with("Let me check that.")); assert!(text.contains("[Called tool `search`")); } @@ -1573,7 +1645,7 @@ mod tests { model: "gpt-4o".to_string(), messages: vec![ChatCompletionMessage { role: "user".to_string(), - content: Some("Hello".to_string()), + content: Some(MessageContent::Text("Hello".to_string())), tool_call_id: None, name: None, tool_calls: None, @@ -1930,7 +2002,7 @@ mod tests { fn test_flatten_tool_result_missing_name_uses_unknown() { let messages = vec![ChatCompletionMessage { role: "tool".to_string(), - content: Some("result data".to_string()), + content: Some(MessageContent::Text("result data".to_string())), tool_call_id: Some("call_1".to_string()), name: None, tool_calls: None, @@ -1942,6 +2014,8 @@ mod tests { .content .as_ref() .unwrap() + .as_text() + .unwrap() .contains("[Tool `unknown` returned:") ); } @@ -1962,6 +2036,8 @@ mod tests { .content .as_ref() .unwrap() + .as_text() + .unwrap() .contains("[Tool `my_tool` returned: ]") ); } @@ -1995,14 +2071,14 @@ mod tests { }, ChatCompletionMessage { role: "tool".to_string(), - content: Some("found".to_string()), + content: Some(MessageContent::Text("found".to_string())), tool_call_id: Some("call_1".to_string()), name: Some("search".to_string()), tool_calls: None, }, ChatCompletionMessage { role: "tool".to_string(), - content: Some("fetched".to_string()), + content: Some(MessageContent::Text("fetched".to_string())), tool_call_id: Some("call_2".to_string()), name: Some("fetch".to_string()), tool_calls: None, @@ -2011,7 +2087,7 @@ mod tests { let result = flatten_tool_messages(messages); assert_eq!(result.len(), 3); // Assistant message has both calls described - let assistant_text = result[0].content.as_ref().unwrap(); + let assistant_text = result[0].content.as_ref().unwrap().as_text().unwrap(); assert!(assistant_text.contains("[Called tool `search`")); assert!(assistant_text.contains("[Called tool `fetch`")); assert!(result[0].tool_calls.is_none()); @@ -2047,8 +2123,8 @@ mod tests { let chat_msg: ChatCompletionMessage = msg.into(); assert_eq!(chat_msg.role, "system"); assert_eq!( - chat_msg.content, - Some("You are a helpful assistant.".to_string()) + chat_msg.content.as_ref().unwrap().as_text().unwrap(), + "You are a helpful assistant." ); assert!(chat_msg.tool_calls.is_none()); assert!(chat_msg.tool_call_id.is_none()); diff --git a/src/llm/provider.rs b/src/llm/provider.rs index 0bcdd4ea..83863573 100644 --- a/src/llm/provider.rs +++ b/src/llm/provider.rs @@ -16,11 +16,38 @@ pub enum Role { Tool, } +/// A part of multimodal message content (OpenAI Chat Completions format). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum ContentPart { + /// Text content part. + #[serde(rename = "text")] + Text { text: String }, + /// Image URL content part (supports data: URLs for inline base64 images). + #[serde(rename = "image_url")] + ImageUrl { image_url: ImageUrl }, +} + +/// Image URL reference for multimodal content. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ImageUrl { + /// URL or data: URI (e.g., "data:image/jpeg;base64,..."). + pub url: String, + /// Detail level hint: "auto", "low", or "high". + #[serde(skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + /// A message in a conversation. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ChatMessage { pub role: Role, pub content: String, + /// Multimodal content parts (images, etc.). + /// When non-empty, providers serialize content as an array of parts + /// (with `content` included as a text part) instead of a plain string. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub content_parts: Vec, /// Tool call ID if this is a tool result message. #[serde(skip_serializing_if = "Option::is_none")] pub tool_call_id: Option, @@ -39,6 +66,7 @@ impl ChatMessage { Self { role: Role::System, content: content.into(), + content_parts: Vec::new(), tool_call_id: None, name: None, tool_calls: None, @@ -50,6 +78,21 @@ impl ChatMessage { Self { role: Role::User, content: content.into(), + content_parts: Vec::new(), + tool_call_id: None, + name: None, + tool_calls: None, + } + } + + /// Create a user message with multimodal content parts (e.g., images). + /// + /// The text `content` is included as the primary text alongside the parts. + pub fn user_with_parts(content: impl Into, parts: Vec) -> Self { + Self { + role: Role::User, + content: content.into(), + content_parts: parts, tool_call_id: None, name: None, tool_calls: None, @@ -61,6 +104,7 @@ impl ChatMessage { Self { role: Role::Assistant, content: content.into(), + content_parts: Vec::new(), tool_call_id: None, name: None, tool_calls: None, @@ -75,6 +119,7 @@ impl ChatMessage { Self { role: Role::Assistant, content: content.unwrap_or_default(), + content_parts: Vec::new(), tool_call_id: None, name: None, tool_calls: if tool_calls.is_empty() { @@ -94,6 +139,7 @@ impl ChatMessage { Self { role: Role::Tool, content: content.into(), + content_parts: Vec::new(), tool_call_id: Some(tool_call_id.into()), name: Some(name.into()), tool_calls: None, diff --git a/src/llm/rig_adapter.rs b/src/llm/rig_adapter.rs index 3253b961..d72373e6 100644 --- a/src/llm/rig_adapter.rs +++ b/src/llm/rig_adapter.rs @@ -11,8 +11,9 @@ use rig::completion::{ ToolDefinition as RigToolDefinition, Usage as RigUsage, }; use rig::message::{ - Message as RigMessage, ToolChoice as RigToolChoice, ToolFunction, ToolResult as RigToolResult, - ToolResultContent, UserContent, + DocumentSourceKind, Image, ImageMediaType, Message as RigMessage, MimeType, + ToolChoice as RigToolChoice, ToolFunction, ToolResult as RigToolResult, ToolResultContent, + UserContent, }; use rust_decimal::Decimal; use rust_decimal_macros::dec; @@ -264,7 +265,41 @@ fn convert_messages(messages: &[ChatMessage]) -> (Option, Vec { - history.push(RigMessage::user(&msg.content)); + if msg.content_parts.is_empty() { + history.push(RigMessage::user(&msg.content)); + } else { + // Build multimodal user message with text + image parts + let mut contents: Vec = vec![UserContent::text(&msg.content)]; + for part in &msg.content_parts { + if let crate::llm::ContentPart::ImageUrl { image_url } = part { + // Parse data: URL for base64 images, or use raw URL + let image = if let Some(rest) = image_url.url.strip_prefix("data:") { + // Format: data:;base64, + let (mime, b64) = + rest.split_once(";base64,").unwrap_or(("image/jpeg", rest)); + Image { + data: DocumentSourceKind::base64(b64), + media_type: ImageMediaType::from_mime_type(mime), + detail: None, + additional_params: None, + } + } else { + Image { + data: DocumentSourceKind::url(&image_url.url), + media_type: None, + detail: None, + additional_params: None, + } + }; + contents.push(UserContent::Image(image)); + } + } + if let Ok(many) = OneOrMany::many(contents) { + history.push(RigMessage::User { content: many }); + } else { + history.push(RigMessage::user(&msg.content)); + } + } } crate::llm::Role::Assistant => { if let Some(ref tool_calls) = msg.tool_calls { @@ -761,6 +796,7 @@ mod tests { let messages = vec![ChatMessage { role: crate::llm::Role::Tool, content: "result text".to_string(), + content_parts: Vec::new(), tool_call_id: None, name: Some("search".to_string()), tool_calls: None, @@ -910,6 +946,7 @@ mod tests { let tool_result_msg = ChatMessage { role: crate::llm::Role::Tool, content: "search results here".to_string(), + content_parts: Vec::new(), tool_call_id: None, name: Some("search".to_string()), tool_calls: None, diff --git a/src/main.rs b/src/main.rs index 54869afe..f50eb754 100644 --- a/src/main.rs +++ b/src/main.rs @@ -669,6 +669,13 @@ async fn async_main() -> anyhow::Result<()> { cost_guard: components.cost_guard, sse_tx: sse_sender, http_interceptor, + transcription: config + .transcription + .create_provider() + .map(|p| Arc::new(ironclaw::transcription::TranscriptionMiddleware::new(p))), + document_extraction: Some(Arc::new( + ironclaw::document_extraction::DocumentExtractionMiddleware::new(), + )), }; let agent = Agent::new( @@ -1107,6 +1114,9 @@ fn check_onboard_needed() -> Option<&'static str> { /// /// Looks for secrets matching the pattern `{channel_name}_*` and injects them /// as credential placeholders (e.g., `telegram_bot_token` -> `{TELEGRAM_BOT_TOKEN}`). +/// +/// Falls back to environment variables with the uppercase name if not found +/// in the secrets store (e.g., `TELEGRAM_BOT_TOKEN`). async fn inject_channel_credentials( channel: &Arc, secrets: &dyn SecretsStore, @@ -1119,6 +1129,7 @@ async fn inject_channel_credentials( let prefix = format!("{}_", channel_name); let mut count = 0; + let mut injected_placeholders = std::collections::HashSet::new(); for secret_meta in all_secrets { if !secret_meta.name.starts_with(&prefix) { @@ -1149,8 +1160,33 @@ async fn inject_channel_credentials( channel .set_credential(&placeholder, decrypted.expose().to_string()) .await; + injected_placeholders.insert(placeholder); count += 1; } + // Fall back to environment variables for required secrets not found in the store. + // This allows channels to work when configured via env vars (e.g., TELEGRAM_BOT_TOKEN) + // without requiring the setup wizard to have run. + let caps = channel.capabilities(); + if let Some(ref http_cap) = caps.tool_capabilities.http { + for cred_mapping in http_cap.credentials.values() { + let placeholder = cred_mapping.secret_name.to_uppercase(); + if injected_placeholders.contains(&placeholder) { + continue; + } + if let Ok(env_value) = std::env::var(&placeholder) + && !env_value.is_empty() + { + tracing::debug!( + channel = %channel_name, + placeholder = %placeholder, + "Injecting credential from environment variable" + ); + channel.set_credential(&placeholder, env_value).await; + count += 1; + } + } + } + Ok(count) } diff --git a/src/registry/manifest.rs b/src/registry/manifest.rs index 495bb8c6..a000442a 100644 --- a/src/registry/manifest.rs +++ b/src/registry/manifest.rs @@ -195,6 +195,7 @@ impl ExtensionManifest { source, fallback_source, auth_hint, + version: Some(self.version.clone()), } } } diff --git a/src/settings.rs b/src/settings.rs index 5ae6c7e8..fb262523 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -99,6 +99,10 @@ pub struct Settings { /// Builder configuration. #[serde(default)] pub builder: BuilderSettings, + + /// Transcription configuration. + #[serde(default)] + pub transcription: Option, } /// Source for the secrets master key. @@ -600,6 +604,14 @@ impl Default for BuilderSettings { } } +/// Transcription pipeline settings. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TranscriptionSettings { + /// Whether audio transcription is enabled. + #[serde(default)] + pub enabled: bool, +} + impl Settings { /// Reconstruct Settings from a flat key-value map (as stored in the DB). /// diff --git a/src/testing.rs b/src/testing.rs index 01c7fdf1..8660f82f 100644 --- a/src/testing.rs +++ b/src/testing.rs @@ -451,6 +451,8 @@ impl TestHarnessBuilder { cost_guard, sse_tx: None, http_interceptor: None, + transcription: None, + document_extraction: None, }; TestHarness { diff --git a/src/tools/builtin/extension_tools.rs b/src/tools/builtin/extension_tools.rs index bb0d9780..7ba4ef0c 100644 --- a/src/tools/builtin/extension_tools.rs +++ b/src/tools/builtin/extension_tools.rs @@ -496,6 +496,68 @@ impl Tool for ToolRemoveTool { } } +// ── tool_upgrade ───────────────────────────────────────────────────── + +pub struct ToolUpgradeTool { + manager: Arc, +} + +impl ToolUpgradeTool { + pub fn new(manager: Arc) -> Self { + Self { manager } + } +} + +#[async_trait] +impl Tool for ToolUpgradeTool { + fn name(&self) -> &str { + "tool_upgrade" + } + + fn description(&self) -> &str { + "Upgrade installed WASM extensions (channels and tools) to match the current \ + host WIT version. If name is omitted, checks and upgrades all installed WASM \ + extensions. Authentication and secrets are preserved." + } + + fn parameters_schema(&self) -> serde_json::Value { + serde_json::json!({ + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Extension name to upgrade (omit to upgrade all)" + } + } + }) + } + + async fn execute( + &self, + params: serde_json::Value, + _ctx: &JobContext, + ) -> Result { + let start = std::time::Instant::now(); + + let name = params.get("name").and_then(|v| v.as_str()); + + let result = self + .manager + .upgrade(name) + .await + .map_err(|e| ToolError::ExecutionFailed(e.to_string()))?; + + let output = serde_json::to_value(&result) + .unwrap_or_else(|_| serde_json::json!({"error": "serialization failed"})); + + Ok(ToolOutput::success(output, start.elapsed())) + } + + fn requires_approval(&self, _params: &serde_json::Value) -> ApprovalRequirement { + ApprovalRequirement::UnlessAutoApproved + } +} + // ── extension_info ──────────────────────────────────────────────────── pub struct ExtensionInfoTool { @@ -643,6 +705,26 @@ mod tests { ); } + #[test] + fn test_tool_upgrade_schema() { + use crate::tools::tool::ApprovalRequirement; + let tool = ToolUpgradeTool { + manager: test_manager_stub(), + }; + assert_eq!(tool.name(), "tool_upgrade"); + assert_eq!( + tool.requires_approval(&serde_json::json!({})), + ApprovalRequirement::UnlessAutoApproved + ); + let schema = tool.parameters_schema(); + // name is optional (omit to upgrade all) + assert!(schema["properties"].get("name").is_some()); + assert!( + schema.get("required").is_none(), + "tool_upgrade should have no required params" + ); + } + #[test] fn test_extension_info_schema() { let tool = ExtensionInfoTool { diff --git a/src/tools/builtin/http.rs b/src/tools/builtin/http.rs index 0fbdd1de..c6e09139 100644 --- a/src/tools/builtin/http.rs +++ b/src/tools/builtin/http.rs @@ -1,12 +1,4 @@ //! HTTP request tool. -//! -//! Unified HTTP tool that handles both simple page/API fetches (GET, no auth) -//! and full API calls (any method, custom headers, credential injection). -//! -//! - Plain GET without auth headers/body → no approval needed, follows redirects -//! - Everything else → requires approval -//! -//! Replaces the former `web_fetch` tool which was a separate GET-only tool. use std::collections::HashMap; use std::net::{IpAddr, ToSocketAddrs}; @@ -26,22 +18,18 @@ use crate::tools::wasm::{InjectedCredentials, SharedCredentialRegistry, inject_c #[cfg(feature = "html-to-markdown")] use crate::tools::builtin::convert_html_to_markdown; -/// Maximum response body size (5 MB). +/// Maximum response body size for text responses (5 MB). /// /// 5 MB is large enough for typical JSON API responses and moderate HTML pages, /// but small enough to prevent OOM from malicious or runaway servers. The WASM /// HTTP wrapper uses the same limit for consistency. const MAX_RESPONSE_SIZE: usize = 5 * 1024 * 1024; -/// Maximum number of redirects to follow for simple GET requests. -const MAX_REDIRECTS: usize = 3; - -/// Descriptive User-Agent so public APIs don't reject bare requests. -const USER_AGENT: &str = concat!( - "IronClaw-Agent/", - env!("CARGO_PKG_VERSION"), - " (https://github.com/nearai/ironclaw)" -); +/// Maximum response body size when saving to disk via `save_to` (50 MB). +/// +/// Larger limit for file downloads since the body is written to disk, not held +/// in memory for LLM context. Matches the WASM attachment size cap. +const MAX_SAVE_TO_SIZE: usize = 50 * 1024 * 1024; /// Tool for making HTTP requests. pub struct HttpTool { @@ -55,8 +43,45 @@ impl HttpTool { pub fn new() -> Self { let client = Client::builder() .timeout(Duration::from_secs(30)) - .redirect(reqwest::redirect::Policy::none()) - .user_agent(USER_AGENT) + .redirect(reqwest::redirect::Policy::custom(|attempt| { + if attempt.previous().len() >= 10 { + return attempt.error("too many redirects"); + } + // Reject scheme downgrades (https → http) + if attempt.url().scheme() != "https" { + return attempt.error("redirect to non-HTTPS URL is not allowed"); + } + // Extract host info before consuming attempt + let host_owned = attempt.url().host_str().map(|h| h.to_owned()); + let port = attempt.url().port_or_known_default().unwrap_or(443); + + if let Some(host) = host_owned { + let host_lower = host.to_lowercase(); + if host_lower == "localhost" || host_lower.ends_with(".localhost") { + return attempt.error("redirect to localhost is not allowed"); + } + if let Ok(ip) = host.parse::() + && is_disallowed_ip(&ip) + { + return attempt.error("redirect to private/local IP is not allowed"); + } + // Resolve hostname and check all IPs + let socket_addr = format!("{}:{}", host, port); + if let Ok(addrs) = socket_addr.to_socket_addrs() { + for addr in addrs { + if is_disallowed_ip(&addr.ip()) { + let msg = format!( + "redirect target '{}' resolves to disallowed IP {}", + host, + addr.ip() + ); + return attempt.error(msg); + } + } + } + } + attempt.follow() + })) .build() .expect("Failed to create HTTP client"); @@ -79,6 +104,31 @@ impl HttpTool { } } +/// Validate and resolve a `save_to` path, ensuring it stays under `/tmp/`. +/// +/// Uses `path_utils::validate_path` with `/tmp` as the base directory to catch +/// traversal attacks like `/tmp/../../etc/passwd` and symlink escapes. +/// Creates parent directories only after validation succeeds. +fn validate_save_to_path(save_to: &str) -> Result { + // Quick prefix check before doing any fs work + if !save_to.starts_with("/tmp/") { + return Err(ToolError::InvalidParameters( + "save_to path must be under /tmp/".to_string(), + )); + } + // Validate path BEFORE creating directories to prevent traversal-based + // directory creation outside /tmp (e.g. `/tmp/../../etc/passwd`). + let tmp_base = std::path::Path::new("/tmp"); + let validated = crate::tools::builtin::path_utils::validate_path(save_to, Some(tmp_base))?; + // Only create parent directories for the validated (safe) path + if let Some(parent) = validated.parent() { + std::fs::create_dir_all(parent).map_err(|e| { + ToolError::ExecutionFailed(format!("failed to create directory: {}", e)) + })?; + } + Ok(validated) +} + pub(crate) fn validate_url(url: &str) -> Result { let parsed = reqwest::Url::parse(url) .map_err(|e| ToolError::InvalidParameters(format!("invalid URL: {}", e)))?; @@ -220,10 +270,9 @@ impl Tool for HttpTool { } fn description(&self) -> &str { - "Make HTTP requests. Simple GET requests (no auth, no custom headers) run without \ - approval and follow redirects — use for fetching weather, public JSON APIs, web pages, \ - and documentation. Requests with authentication, custom headers, or non-GET methods \ - (POST, PUT, DELETE, PATCH) require user approval." + "Make HTTP requests to external APIs. Supports GET, POST, PUT, DELETE methods. \ + Use save_to to download binary files (images, PDFs, etc.) to a local path, \ + e.g. {\"method\":\"GET\",\"url\":\"https://picsum.photos/800/600\",\"save_to\":\"/tmp/photo.jpg\"}." } fn parameters_schema(&self) -> serde_json::Value { @@ -258,6 +307,10 @@ impl Tool for HttpTool { "timeout_secs": { "type": "integer", "description": "Request timeout in seconds (default: 30)" + }, + "save_to": { + "type": "string", + "description": "Save response body as raw bytes to this file path instead of returning it. Use for binary downloads (images, PDFs, etc.). The path must be under /tmp/." } }, "required": ["method", "url"] @@ -390,130 +443,50 @@ impl Tool for HttpTool { return Ok(ToolOutput::success(result, start.elapsed()).with_raw(recorded.body)); } - // Determine if this is a simple GET (eligible for redirect following). - let is_simple_get = - method.eq_ignore_ascii_case("GET") && headers_vec.is_empty() && body_bytes.is_none(); - - // Execute request, optionally following redirects for simple GETs. - let response = if is_simple_get { - let mut redirects_remaining = MAX_REDIRECTS; - loop { - let resp = self - .client - .get(parsed_url.clone()) - .header( - reqwest::header::ACCEPT, - "text/markdown, text/html;q=0.9, application/json;q=0.9, */*;q=0.8", - ) - .send() - .await - .map_err(|e| { - if e.is_timeout() { - ToolError::Timeout(Duration::from_secs(30)) - } else { - ToolError::ExternalService(e.to_string()) - } - })?; - - let status = resp.status().as_u16(); - if (300..400).contains(&status) { - if redirects_remaining == 0 { - return Err(ToolError::ExecutionFailed(format!( - "too many redirects (max {})", - MAX_REDIRECTS - ))); - } - - let location = resp - .headers() - .get(reqwest::header::LOCATION) - .and_then(|v| v.to_str().ok()) - .ok_or_else(|| { - ToolError::ExecutionFailed(format!( - "redirect (HTTP {}) has no Location header", - status - )) - })?; - - let next_url_str = - if location.starts_with("http://") || location.starts_with("https://") { - location.to_string() - } else { - parsed_url - .join(location) - .map(|u| u.to_string()) - .map_err(|e| { - ToolError::ExecutionFailed(format!( - "could not resolve relative redirect '{}': {}", - location, e - )) - })? - }; - - // SSRF re-validation on every hop. - parsed_url = validate_url(&next_url_str)?; - let detector = LeakDetector::new(); - detector - .scan_http_request(parsed_url.as_str(), &[], None) - .map_err(|e| ToolError::NotAuthorized(e.to_string()))?; - - redirects_remaining -= 1; - tracing::debug!( - to = %parsed_url, - hops_left = redirects_remaining, - "http tool following redirect" - ); - continue; - } - - break resp; + // Execute request + let response = request.send().await.map_err(|e| { + if e.is_timeout() { + ToolError::Timeout(Duration::from_secs(30)) + } else { + ToolError::ExternalService(e.to_string()) } - } else { - let resp = request.send().await.map_err(|e| { - if e.is_timeout() { - ToolError::Timeout(Duration::from_secs(30)) - } else { - ToolError::ExternalService(e.to_string()) - } - })?; - - let status = resp.status().as_u16(); - - // Block redirects for non-simple requests (potential SSRF) - if (300..400).contains(&status) { - return Err(ToolError::NotAuthorized(format!( - "request returned redirect (HTTP {}), which is blocked to prevent SSRF", - status - ))); - } - - resp - }; + })?; let status = response.status().as_u16(); + // Redirects are followed automatically (up to 10 hops). + // If we still see a 3xx here, the chain was too long. + let headers: HashMap = response .headers() .iter() .filter_map(|(k, v)| v.to_str().ok().map(|v| (k.to_string(), v.to_string()))) .collect(); + // Use a larger size limit when saving to disk (file downloads) + let saving_to_disk = params.get("save_to").is_some(); + let max_size = if saving_to_disk { + MAX_SAVE_TO_SIZE + } else { + MAX_RESPONSE_SIZE + }; + // Pre-check Content-Length header to reject obviously oversized responses // before downloading anything, preventing OOM from malicious servers. if let Some(content_length) = response.headers().get(reqwest::header::CONTENT_LENGTH) && let Ok(s) = content_length.to_str() && let Ok(len) = s.parse::() - && len > MAX_RESPONSE_SIZE + && len > max_size { tracing::warn!( url = %parsed_url, content_length = len, - max = MAX_RESPONSE_SIZE, + max = max_size, "Rejected HTTP response: Content-Length exceeds limit" ); return Err(ToolError::ExecutionFailed(format!( "Response Content-Length ({} bytes) exceeds maximum allowed size ({} bytes)", - len, MAX_RESPONSE_SIZE + len, max_size ))); } @@ -525,16 +498,39 @@ impl Tool for HttpTool { let chunk = chunk.map_err(|e| { ToolError::ExternalService(format!("failed to read response body: {}", e)) })?; - if body.len() + chunk.len() > MAX_RESPONSE_SIZE { + if body.len() + chunk.len() > max_size { return Err(ToolError::ExecutionFailed(format!( "Response body exceeds maximum allowed size ({} bytes)", - MAX_RESPONSE_SIZE + max_size ))); } body.extend_from_slice(&chunk); } let body_bytes = bytes::Bytes::from(body); + // If save_to is specified, write raw bytes to file and return metadata. + if let Some(save_to) = params.get("save_to").and_then(|v| v.as_str()) { + let save_to_owned = save_to.to_string(); + let bytes_clone = body_bytes.clone(); + tokio::task::spawn_blocking(move || { + let canonical = validate_save_to_path(&save_to_owned)?; + std::fs::write(&canonical, &bytes_clone).map_err(|e| { + ToolError::ExecutionFailed(format!("failed to write file: {}", e)) + })?; + Ok::<_, ToolError>(canonical) + }) + .await + .map_err(|e| ToolError::ExecutionFailed(format!("spawn_blocking failed: {}", e)))? + .map_err(|e: ToolError| e)?; + let result = serde_json::json!({ + "status": status, + "saved_to": save_to, + "size_bytes": body_bytes.len(), + "headers": headers, + }); + return Ok(ToolOutput::success(result, start.elapsed())); + } + let body_text = String::from_utf8_lossy(&body_bytes).into_owned(); // Record the HTTP exchange if interceptor is present (recording mode) @@ -601,25 +597,6 @@ impl Tool for HttpTool { { return ApprovalRequirement::Always; } - // 3. Plain GET without headers or body → no approval needed - let method = params - .get("method") - .and_then(|v| v.as_str()) - .unwrap_or("GET"); - let has_headers = params - .get("headers") - .map(|h| match h { - serde_json::Value::Array(a) => !a.is_empty(), - serde_json::Value::Object(o) => !o.is_empty(), - _ => false, - }) - .unwrap_or(false); - let has_body = params.get("body").is_some(); - - if method.eq_ignore_ascii_case("GET") && !has_headers && !has_body { - return ApprovalRequirement::Never; - } - // Default: outbound HTTP still needs approval unless auto-approved ApprovalRequirement::UnlessAutoApproved } @@ -746,37 +723,12 @@ mod tests { // ── Approval requirement tests ────────────────────────────────────── #[test] - fn test_plain_get_returns_never() { + fn test_no_auth_headers_returns_unless_auto_approved() { let tool = HttpTool::new(); let params = serde_json::json!({ "method": "GET", "url": "https://api.example.com/data" }); - assert_eq!(tool.requires_approval(¶ms), ApprovalRequirement::Never); - } - - #[test] - fn test_post_returns_unless_auto_approved() { - let tool = HttpTool::new(); - let params = serde_json::json!({ - "method": "POST", - "url": "https://api.example.com/data", - "body": {"key": "value"} - }); - assert_eq!( - tool.requires_approval(¶ms), - ApprovalRequirement::UnlessAutoApproved - ); - } - - #[test] - fn test_get_with_headers_returns_unless_auto_approved() { - let tool = HttpTool::new(); - let params = serde_json::json!({ - "method": "GET", - "url": "https://api.example.com/data", - "headers": [{"name": "X-Custom", "value": "test"}] - }); assert_eq!( tool.requires_approval(¶ms), ApprovalRequirement::UnlessAutoApproved @@ -874,24 +826,30 @@ mod tests { } #[test] - fn test_empty_headers_get_returns_never() { + fn test_empty_headers_return_unless_auto_approved() { let tool = HttpTool::new(); - // Empty object — still a plain GET + // Empty object let params = serde_json::json!({ "method": "GET", "url": "https://example.com", "headers": {} }); - assert_eq!(tool.requires_approval(¶ms), ApprovalRequirement::Never); + assert_eq!( + tool.requires_approval(¶ms), + ApprovalRequirement::UnlessAutoApproved + ); - // Empty array — still a plain GET + // Empty array let params = serde_json::json!({ "method": "GET", "url": "https://example.com", "headers": [] }); - assert_eq!(tool.requires_approval(¶ms), ApprovalRequirement::Never); + assert_eq!( + tool.requires_approval(¶ms), + ApprovalRequirement::UnlessAutoApproved + ); } // ── Credential registry approval tests ───────────────────────────── @@ -926,7 +884,7 @@ mod tests { } #[test] - fn test_host_without_credential_mapping_get_returns_never() { + fn test_host_without_credential_mapping_returns_unless_auto_approved() { use crate::tools::wasm::SharedCredentialRegistry; let registry = Arc::new(SharedCredentialRegistry::new()); @@ -942,19 +900,10 @@ mod tests { ))), ); - // Plain GET with no credentials → Never let params = serde_json::json!({ "method": "GET", "url": "https://api.example.com/data" }); - assert_eq!(tool.requires_approval(¶ms), ApprovalRequirement::Never); - - // POST with no credentials → UnlessAutoApproved - let params = serde_json::json!({ - "method": "POST", - "url": "https://api.example.com/data", - "body": {"key": "value"} - }); assert_eq!( tool.requires_approval(¶ms), ApprovalRequirement::UnlessAutoApproved @@ -1038,4 +987,60 @@ mod tests { }); let _ = tool.requires_approval(¶ms_with_auth); } + + // ── save_to path validation tests ───────────────────────────────────── + + #[test] + fn test_save_to_rejects_path_outside_tmp() { + let err = validate_save_to_path("/etc/passwd").unwrap_err(); + assert!(err.to_string().contains("must be under /tmp/")); + } + + #[test] + fn test_save_to_rejects_home_dir() { + let err = validate_save_to_path("/home/user/file.txt").unwrap_err(); + assert!(err.to_string().contains("must be under /tmp/")); + } + + #[test] + fn test_save_to_rejects_traversal_via_dotdot() { + let err = validate_save_to_path("/tmp/../../etc/passwd").unwrap_err(); + let msg = err.to_string(); + assert!( + msg.contains("escapes") || msg.contains("resolves outside"), + "expected path traversal rejection, got: {}", + msg + ); + } + + #[test] + fn test_save_to_rejects_deep_traversal() { + let err = validate_save_to_path("/tmp/a/b/../../../../etc/shadow").unwrap_err(); + let msg = err.to_string(); + assert!( + msg.contains("escapes") || msg.contains("resolves outside"), + "expected path traversal rejection, got: {}", + msg + ); + } + + #[test] + fn test_save_to_accepts_simple_tmp_path() { + let path = validate_save_to_path("/tmp/test_ironclaw_photo.jpg").unwrap(); + assert!(path.starts_with("/tmp")); + let _ = std::fs::remove_file(&path); + } + + #[test] + fn test_save_to_accepts_nested_tmp_path() { + let path = validate_save_to_path("/tmp/ironclaw_test_subdir/nested/file.png").unwrap(); + assert!(path.starts_with("/tmp")); + let _ = std::fs::remove_dir_all("/tmp/ironclaw_test_subdir"); + } + + #[test] + fn test_save_to_rejects_bare_tmp() { + let err = validate_save_to_path("/tmp").unwrap_err(); + assert!(err.to_string().contains("must be under /tmp/")); + } } diff --git a/src/tools/builtin/message.rs b/src/tools/builtin/message.rs index 9e37da6c..4259d3dd 100644 --- a/src/tools/builtin/message.rs +++ b/src/tools/builtin/message.rs @@ -68,6 +68,9 @@ impl Tool for MessageTool { fn description(&self) -> &str { "Send a message to a channel. If channel/target omitted, uses the current conversation's \ channel and sender/group. Use to proactively message users on any connected channel. \ + Supports file attachments: first download the file with the http tool using save_to \ + (e.g., http GET https://picsum.photos/800/600 save_to=/tmp/photo.jpg), then pass \ + the file path in the attachments array. Images are sent as photos on Telegram. \ - Signal: target accepts E.164 (+1234567890) or group ID \ - Telegram: target accepts username or chat ID \ - Slack: target accepts channel (#general) or user ID" @@ -149,13 +152,18 @@ impl Tool for MessageTool { let attachment_count = attachments.len(); - // Validate all attachment paths against the sandbox and verify existence + // Validate all attachment paths against the sandbox and verify existence. + // Allow paths under the base_dir (~/.ironclaw) or /tmp/. for path in &attachments { + let tmp_dir = PathBuf::from("/tmp"); let resolved = crate::tools::builtin::path_utils::validate_path(path, Some(&self.base_dir)) + .or_else(|_| { + crate::tools::builtin::path_utils::validate_path(path, Some(&tmp_dir)) + }) .map_err(|e| { ToolError::ExecutionFailed(format!( - "Attachment path must be within {}: {}", + "Attachment path must be within {} or /tmp/: {}", self.base_dir.display(), e )) @@ -325,22 +333,24 @@ mod tests { tool.set_context(Some("signal".to_string()), Some("+1234567890".to_string())) .await; - // Execute with attachments outside sandbox + // Execute with attachments outside both sandbox (~/.ironclaw) and /tmp/ let ctx = crate::context::JobContext::new("test", "test description"); let result = tool .execute( serde_json::json!({ "content": "hello", - "attachments": ["/tmp/file1.txt", "/tmp/file2.png"] + "attachments": ["/etc/passwd", "/var/log/syslog"] }), &ctx, ) .await; - // Should fail due to sandbox rejection (paths outside ~/.ironclaw/) + // Should fail due to sandbox rejection (paths outside allowed directories) assert!(result.is_err()); let err = result.unwrap_err().to_string(); - assert!(err.contains("sandbox") || err.contains("escapes")); + assert!( + err.contains("sandbox") || err.contains("escapes") || err.contains("must be within"), + ); } #[tokio::test] @@ -376,6 +386,42 @@ mod tests { assert!(err.contains("channel") || err.contains("Channel")); } + #[tokio::test] + async fn message_tool_with_attachments_in_tmp_no_channel() { + use std::fs; + + let tool = MessageTool::new(Arc::new(ChannelManager::new())); + tool.set_context(Some("telegram".to_string()), Some("12345".to_string())) + .await; + + // Create temp files under /tmp (allowed as secondary attachment dir) + let temp_dir = tempfile::tempdir_in("/tmp").unwrap(); + let file1 = temp_dir.path().join("photo.jpg"); + let file2 = temp_dir.path().join("doc.pdf"); + fs::write(&file1, "fake image data").unwrap(); + fs::write(&file2, "fake pdf data").unwrap(); + + let ctx = crate::context::JobContext::new("test", "test description"); + let result = tool + .execute( + serde_json::json!({ + "content": "here are the files", + "attachments": [file1.to_string_lossy(), file2.to_string_lossy()] + }), + &ctx, + ) + .await; + + // Path validation passes for /tmp paths, fails at channel send (no real channel) + assert!(result.is_err()); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("channel") || err.contains("Channel"), + "expected channel error (path validation should pass), got: {}", + err + ); + } + #[tokio::test] async fn message_tool_requires_content() { let tool = MessageTool::new(Arc::new(ChannelManager::new())); diff --git a/src/tools/builtin/mod.rs b/src/tools/builtin/mod.rs index 23f170f9..bbbc7056 100644 --- a/src/tools/builtin/mod.rs +++ b/src/tools/builtin/mod.rs @@ -19,7 +19,7 @@ mod time; pub use echo::EchoTool; pub use extension_tools::{ ExtensionInfoTool, ToolActivateTool, ToolAuthTool, ToolInstallTool, ToolListTool, - ToolRemoveTool, ToolSearchTool, + ToolRemoveTool, ToolSearchTool, ToolUpgradeTool, }; pub use file::{ApplyPatchTool, ListDirTool, ReadFileTool, WriteFileTool}; pub use http::HttpTool; diff --git a/src/tools/registry.rs b/src/tools/registry.rs index 4f98a30b..498d1d58 100644 --- a/src/tools/registry.rs +++ b/src/tools/registry.rs @@ -21,7 +21,7 @@ use crate::tools::builtin::{ MemoryReadTool, MemorySearchTool, MemoryTreeTool, MemoryWriteTool, PromptQueue, ReadFileTool, ShellTool, SkillInstallTool, SkillListTool, SkillRemoveTool, SkillSearchTool, TimeTool, ToolActivateTool, ToolAuthTool, ToolInstallTool, ToolListTool, ToolRemoveTool, ToolSearchTool, - WriteFileTool, + ToolUpgradeTool, WriteFileTool, }; use crate::tools::rate_limiter::RateLimiter; use crate::tools::tool::{Tool, ToolDomain}; @@ -389,8 +389,9 @@ impl ToolRegistry { self.register_sync(Arc::new(ToolActivateTool::new(Arc::clone(&manager)))); self.register_sync(Arc::new(ToolListTool::new(Arc::clone(&manager)))); self.register_sync(Arc::new(ToolRemoveTool::new(Arc::clone(&manager)))); + self.register_sync(Arc::new(ToolUpgradeTool::new(Arc::clone(&manager)))); self.register_sync(Arc::new(ExtensionInfoTool::new(manager))); - tracing::info!("Registered 7 extension management tools"); + tracing::info!("Registered 8 extension management tools"); } /// Register skill management tools (list, search, install, remove). diff --git a/src/tools/wasm/loader.rs b/src/tools/wasm/loader.rs index ab94553e..4a9207b9 100644 --- a/src/tools/wasm/loader.rs +++ b/src/tools/wasm/loader.rs @@ -328,7 +328,7 @@ impl WasmToolLoader { /// - Extension WIT version must not be greater than host version /// /// If `declared` is `None`, the check is skipped (pre-versioning extension). -pub(crate) fn check_wit_version_compat( +pub fn check_wit_version_compat( name: &str, declared: Option<&str>, host_version: &str, diff --git a/src/tools/wasm/mod.rs b/src/tools/wasm/mod.rs index fc3a3939..55b5b0cd 100644 --- a/src/tools/wasm/mod.rs +++ b/src/tools/wasm/mod.rs @@ -77,10 +77,10 @@ /// /// Extensions declaring a `wit_version` in their capabilities file are checked /// against this at load time: same major, not greater than host. -pub const WIT_TOOL_VERSION: &str = "0.2.0"; +pub const WIT_TOOL_VERSION: &str = "0.3.0"; /// Host WIT version for channel extensions. -pub const WIT_CHANNEL_VERSION: &str = "0.2.0"; +pub const WIT_CHANNEL_VERSION: &str = "0.3.0"; mod allowlist; mod capabilities; @@ -131,8 +131,9 @@ pub use storage::{ // Loader pub use loader::{ - DiscoveredTool, LoadResults, WasmLoadError, WasmToolLoader, discover_dev_tools, discover_tools, - load_dev_tools, resolve_wasm_target_dir, wasm_artifact_path, + DiscoveredTool, LoadResults, WasmLoadError, WasmToolLoader, check_wit_version_compat, + discover_dev_tools, discover_tools, load_dev_tools, resolve_wasm_target_dir, + wasm_artifact_path, }; // Capabilities schema (for parsing *.capabilities.json files) diff --git a/src/transcription/mod.rs b/src/transcription/mod.rs new file mode 100644 index 00000000..d0a7d31c --- /dev/null +++ b/src/transcription/mod.rs @@ -0,0 +1,287 @@ +//! Audio transcription pipeline. +//! +//! Provides a [`TranscriptionProvider`] trait for pluggable speech-to-text +//! backends and a [`TranscriptionMiddleware`] that detects audio attachments +//! on incoming messages and replaces them with transcribed text. + +mod openai; + +pub use self::openai::OpenAiWhisperProvider; + +use async_trait::async_trait; + +/// Supported audio formats for transcription. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AudioFormat { + Ogg, + Mp3, + Mp4, + Wav, + Webm, + Flac, + M4a, +} + +impl AudioFormat { + /// Infer audio format from MIME type. Returns `None` for unsupported types. + pub fn from_mime_type(mime: &str) -> Option { + let base = mime.split(';').next().unwrap_or(mime).trim(); + match base { + "audio/ogg" | "audio/opus" => Some(Self::Ogg), + "audio/mpeg" | "audio/mp3" => Some(Self::Mp3), + "audio/mp4" => Some(Self::Mp4), + "audio/wav" | "audio/x-wav" => Some(Self::Wav), + "audio/webm" => Some(Self::Webm), + "audio/flac" | "audio/x-flac" => Some(Self::Flac), + "audio/m4a" | "audio/x-m4a" | "audio/aac" => Some(Self::M4a), + _ => None, + } + } + + /// File extension for this format (used as the filename in multipart uploads). + pub fn extension(&self) -> &'static str { + match self { + Self::Ogg => "ogg", + Self::Mp3 => "mp3", + Self::Mp4 => "mp4", + Self::Wav => "wav", + Self::Webm => "webm", + Self::Flac => "flac", + Self::M4a => "m4a", + } + } +} + +/// Errors from the transcription pipeline. +#[derive(Debug, thiserror::Error)] +pub enum TranscriptionError { + #[error("Transcription request failed: {0}")] + RequestFailed(String), + + #[error("Unsupported audio format: {mime_type}")] + UnsupportedFormat { mime_type: String }, + + #[error("Audio data is empty")] + EmptyAudio, +} + +/// Trait for speech-to-text providers. +#[async_trait] +pub trait TranscriptionProvider: Send + Sync { + /// Transcribe audio bytes into text. + async fn transcribe( + &self, + audio_data: &[u8], + format: AudioFormat, + ) -> Result; +} + +/// Middleware that processes audio attachments on incoming messages. +/// +/// When an incoming message has audio attachments with inline data, +/// the middleware transcribes them and sets `extracted_text` on the attachment. +/// If the message has no text content, the transcription becomes the message content. +pub struct TranscriptionMiddleware { + provider: Box, +} + +impl TranscriptionMiddleware { + /// Create a new middleware with the given transcription provider. + pub fn new(provider: Box) -> Self { + Self { provider } + } + + /// Process an incoming message, transcribing any audio attachments with data. + /// + /// Modifies the message in place: + /// - Sets `extracted_text` on audio attachments that have inline data + /// - If the message content is empty, sets it to the transcription + pub async fn process(&self, msg: &mut crate::channels::IncomingMessage) { + use crate::channels::AttachmentKind; + + let mut transcriptions = Vec::new(); + + for (i, attachment) in msg.attachments.iter().enumerate() { + if attachment.kind != AttachmentKind::Audio { + continue; + } + if attachment.data.is_empty() { + continue; + } + // Already transcribed + if attachment.extracted_text.is_some() { + continue; + } + + let format = match AudioFormat::from_mime_type(&attachment.mime_type) { + Some(f) => f, + None => { + tracing::warn!( + mime = %attachment.mime_type, + "Skipping audio attachment with unsupported format" + ); + continue; + } + }; + + match self.provider.transcribe(&attachment.data, format).await { + Ok(text) => { + tracing::info!( + attachment_id = %attachment.id, + text_len = text.len(), + "Transcribed audio attachment" + ); + transcriptions.push((i, text)); + } + Err(e) => { + tracing::error!( + attachment_id = %attachment.id, + error = %e, + "Failed to transcribe audio attachment" + ); + transcriptions.push((i, format!("[Transcription failed: {}]", e))); + } + } + } + + for (i, text) in &transcriptions { + msg.attachments[*i].extracted_text = Some(text.clone()); + } + + // If message has no text content, use the first successful transcription + if (msg.content.is_empty() || msg.content == "[Voice note]") + && let Some((_, text)) = transcriptions + .iter() + .find(|(_, t)| !t.starts_with("[Transcription failed")) + { + msg.content = text.clone(); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::channels::{AttachmentKind, IncomingAttachment, IncomingMessage}; + + struct MockProvider { + result: Result, + } + + #[async_trait] + impl TranscriptionProvider for MockProvider { + async fn transcribe( + &self, + _audio_data: &[u8], + _format: AudioFormat, + ) -> Result { + match &self.result { + Ok(text) => Ok(text.clone()), + Err(_) => Err(TranscriptionError::RequestFailed("mock error".into())), + } + } + } + + fn voice_attachment(data: Vec) -> IncomingAttachment { + IncomingAttachment { + id: "voice_123".to_string(), + kind: AttachmentKind::Audio, + mime_type: "audio/ogg".to_string(), + filename: Some("voice.ogg".to_string()), + size_bytes: Some(data.len() as u64), + source_url: None, + storage_key: None, + extracted_text: None, + data, + duration_secs: Some(5), + } + } + + #[tokio::test] + async fn middleware_transcribes_audio_attachment() { + let middleware = TranscriptionMiddleware::new(Box::new(MockProvider { + result: Ok("Hello world".to_string()), + })); + + let mut msg = IncomingMessage::new("telegram", "user1", "[Voice note]") + .with_attachments(vec![voice_attachment(vec![1, 2, 3])]); + + middleware.process(&mut msg).await; + + assert_eq!( + msg.attachments[0].extracted_text.as_deref(), + Some("Hello world") + ); + assert_eq!(msg.content, "Hello world"); + } + + #[tokio::test] + async fn middleware_skips_empty_audio_data() { + let middleware = TranscriptionMiddleware::new(Box::new(MockProvider { + result: Ok("Should not be called".to_string()), + })); + + let mut msg = IncomingMessage::new("telegram", "user1", "text message") + .with_attachments(vec![voice_attachment(Vec::new())]); + + middleware.process(&mut msg).await; + + assert!(msg.attachments[0].extracted_text.is_none()); + assert_eq!(msg.content, "text message"); + } + + #[tokio::test] + async fn middleware_skips_already_transcribed() { + let middleware = TranscriptionMiddleware::new(Box::new(MockProvider { + result: Ok("New transcription".to_string()), + })); + + let mut attachment = voice_attachment(vec![1, 2, 3]); + attachment.extracted_text = Some("Already done".to_string()); + + let mut msg = + IncomingMessage::new("telegram", "user1", "").with_attachments(vec![attachment]); + + middleware.process(&mut msg).await; + + assert_eq!( + msg.attachments[0].extracted_text.as_deref(), + Some("Already done") + ); + } + + #[tokio::test] + async fn middleware_preserves_existing_content() { + let middleware = TranscriptionMiddleware::new(Box::new(MockProvider { + result: Ok("Transcription".to_string()), + })); + + let mut msg = IncomingMessage::new("telegram", "user1", "User typed this") + .with_attachments(vec![voice_attachment(vec![1, 2, 3])]); + + middleware.process(&mut msg).await; + + assert_eq!( + msg.attachments[0].extracted_text.as_deref(), + Some("Transcription") + ); + assert_eq!(msg.content, "User typed this"); + } + + #[test] + fn audio_format_from_mime() { + assert_eq!( + AudioFormat::from_mime_type("audio/ogg"), + Some(AudioFormat::Ogg) + ); + assert_eq!( + AudioFormat::from_mime_type("audio/mpeg"), + Some(AudioFormat::Mp3) + ); + assert_eq!( + AudioFormat::from_mime_type("audio/ogg; codecs=opus"), + Some(AudioFormat::Ogg) + ); + assert_eq!(AudioFormat::from_mime_type("image/jpeg"), None); + } +} diff --git a/src/transcription/openai.rs b/src/transcription/openai.rs new file mode 100644 index 00000000..1df8057e --- /dev/null +++ b/src/transcription/openai.rs @@ -0,0 +1,124 @@ +//! OpenAI Whisper transcription provider. + +use async_trait::async_trait; +use reqwest::multipart; +use secrecy::{ExposeSecret, SecretString}; + +use super::{AudioFormat, TranscriptionError, TranscriptionProvider}; + +/// OpenAI Whisper speech-to-text provider. +/// +/// Uses the `/v1/audio/transcriptions` endpoint. +pub struct OpenAiWhisperProvider { + client: reqwest::Client, + api_key: SecretString, + model: String, + base_url: String, +} + +impl OpenAiWhisperProvider { + /// Create a new Whisper provider with the given API key. + pub fn new(api_key: SecretString) -> Self { + Self { + client: match reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(120)) + .build() + { + Ok(c) => c, + Err(e) => { + tracing::error!( + "Failed to build HTTP client with timeout, falling back to default: {e}" + ); + reqwest::Client::default() + } + }, + api_key, + model: "whisper-1".to_string(), + base_url: "https://api.openai.com".to_string(), + } + } + + /// Override the base URL (for proxied or compatible endpoints). + pub fn with_base_url(mut self, base_url: impl Into) -> Self { + let mut url = base_url.into(); + // Normalize: strip trailing slash to avoid double-slash in URL construction + while url.ends_with('/') { + url.pop(); + } + self.base_url = url; + self + } + + /// Override the model name. + pub fn with_model(mut self, model: impl Into) -> Self { + self.model = model.into(); + self + } +} + +#[async_trait] +impl TranscriptionProvider for OpenAiWhisperProvider { + async fn transcribe( + &self, + audio_data: &[u8], + format: AudioFormat, + ) -> Result { + if audio_data.is_empty() { + return Err(TranscriptionError::EmptyAudio); + } + + let filename = format!("audio.{}", format.extension()); + let mime_str = match format { + AudioFormat::Ogg => "audio/ogg", + AudioFormat::Mp3 => "audio/mpeg", + AudioFormat::Mp4 => "audio/mp4", + AudioFormat::Wav => "audio/wav", + AudioFormat::Webm => "audio/webm", + AudioFormat::Flac => "audio/flac", + AudioFormat::M4a => "audio/m4a", + }; + + let file_part = multipart::Part::bytes(audio_data.to_vec()) + .file_name(filename) + .mime_str(mime_str) + .map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?; + + let form = multipart::Form::new() + .text("model", self.model.clone()) + .text("response_format", "text") + .part("file", file_part); + + let url = format!("{}/v1/audio/transcriptions", self.base_url); + + let response = self + .client + .post(&url) + .header( + "Authorization", + format!("Bearer {}", self.api_key.expose_secret()), + ) + .multipart(form) + .send() + .await + .map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?; + + let status = response.status(); + if !status.is_success() { + let body = response + .text() + .await + .unwrap_or_else(|_| "unknown error".to_string()); + return Err(TranscriptionError::RequestFailed(format!( + "HTTP {}: {}", + status, body + ))); + } + + let text = response + .text() + .await + .map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?; + + Ok(text.trim().to_string()) + } +} diff --git a/tests/e2e_attachments.rs b/tests/e2e_attachments.rs new file mode 100644 index 00000000..c7191109 --- /dev/null +++ b/tests/e2e_attachments.rs @@ -0,0 +1,210 @@ +//! E2E tests for attachment processing in the LLM pipeline. +//! +//! Verifies that attachments on incoming messages are augmented into the user +//! text and (for images) passed as multimodal content parts to the LLM. + +#[cfg(feature = "libsql")] +mod support; + +#[cfg(feature = "libsql")] +mod attachment_tests { + use std::time::Duration; + + use crate::support::test_rig::TestRigBuilder; + use crate::support::trace_llm::LlmTrace; + + use ironclaw::channels::{AttachmentKind, IncomingAttachment, IncomingMessage}; + use ironclaw::llm::ContentPart; + + const FIXTURES: &str = concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/llm_traces/spot" + ); + const TIMEOUT: Duration = Duration::from_secs(15); + + fn make_attachment(kind: AttachmentKind) -> IncomingAttachment { + IncomingAttachment { + id: "att-1".to_string(), + kind, + mime_type: "application/octet-stream".to_string(), + filename: None, + size_bytes: None, + source_url: None, + storage_key: None, + extracted_text: None, + data: vec![], + duration_secs: None, + } + } + + /// Audio attachment with transcript reaches the LLM as augmented text. + #[tokio::test] + async fn attachment_audio_transcript_reaches_llm() { + let trace = + LlmTrace::from_file(format!("{FIXTURES}/attachment_audio_transcript.json")).unwrap(); + let rig = TestRigBuilder::new() + .with_trace(trace.clone()) + .build() + .await; + + // Build a message with an audio attachment containing a transcript + let mut att = make_attachment(AttachmentKind::Audio); + att.filename = Some("voice.ogg".to_string()); + att.mime_type = "audio/ogg".to_string(); + att.extracted_text = Some("Hello, can you help me with my project?".to_string()); + att.duration_secs = Some(5); + + let mut msg = IncomingMessage::new("test", "test-user", "Check this voice note"); + msg.attachments.push(att); + + rig.send_incoming(msg).await; + let responses = rig.wait_for_responses(1, TIMEOUT).await; + + // Verify the response was received + assert!( + !responses.is_empty(), + "should receive at least one response" + ); + + // Verify the augmented content reached the LLM + let requests = rig.captured_llm_requests(); + assert!(!requests.is_empty(), "LLM should have been called"); + + let last_request = &requests[requests.len() - 1]; + let last_user_msg = last_request + .iter() + .rev() + .find(|m| matches!(m.role, ironclaw::llm::Role::User)) + .expect("should have a user message"); + + // The augmented text should contain the attachment tags and transcript + assert!( + last_user_msg.content.contains(""), + "user message should contain block, got: {}", + last_user_msg.content.chars().take(200).collect::() + ); + assert!( + last_user_msg + .content + .contains("Hello, can you help me with my project?"), + "user message should contain the transcript" + ); + assert!( + last_user_msg.content.contains("duration=\"5s\""), + "user message should contain duration" + ); + + // Audio attachments should NOT produce image content parts + assert!( + last_user_msg.content_parts.is_empty(), + "audio attachments should not produce image content parts" + ); + + rig.verify_trace_expects(&trace, &responses); + rig.shutdown(); + } + + /// Image attachment with data reaches the LLM with multimodal content parts. + #[tokio::test] + async fn attachment_image_produces_content_parts() { + let trace = LlmTrace::from_file(format!("{FIXTURES}/attachment_image.json")).unwrap(); + let rig = TestRigBuilder::new() + .with_trace(trace.clone()) + .build() + .await; + + // Build a message with an image attachment that has raw data + let mut att = make_attachment(AttachmentKind::Image); + att.filename = Some("screenshot.png".to_string()); + att.mime_type = "image/png".to_string(); + att.size_bytes = Some(1024); + att.data = vec![0x89, 0x50, 0x4E, 0x47]; // PNG magic bytes (fake) + + let mut msg = + IncomingMessage::new("test", "test-user", "What do you see in this screenshot?"); + msg.attachments.push(att); + + rig.send_incoming(msg).await; + let responses = rig.wait_for_responses(1, TIMEOUT).await; + + assert!( + !responses.is_empty(), + "should receive at least one response" + ); + + // Verify multimodal content parts reached the LLM + let requests = rig.captured_llm_requests(); + assert!(!requests.is_empty(), "LLM should have been called"); + + let last_request = &requests[requests.len() - 1]; + let last_user_msg = last_request + .iter() + .rev() + .find(|m| matches!(m.role, ironclaw::llm::Role::User)) + .expect("should have a user message"); + + // Should have image content parts + assert_eq!( + last_user_msg.content_parts.len(), + 1, + "should have exactly one image content part" + ); + + // Verify the content part is an ImageUrl with a data: URI + match &last_user_msg.content_parts[0] { + ContentPart::ImageUrl { image_url } => { + assert!( + image_url.url.starts_with("data:image/png;base64,"), + "image URL should be a base64 data URI, got: {}", + &image_url.url[..image_url.url.len().min(40)] + ); + } + other => panic!("expected ImageUrl content part, got: {:?}", other), + } + + // The text should note the image is sent as visual content + assert!( + last_user_msg + .content + .contains("[Image attached — sent as visual content]"), + "augmented text should note image sent as visual content" + ); + + rig.verify_trace_expects(&trace, &responses); + rig.shutdown(); + } + + /// Message without attachments should have no content_parts and no augmentation. + #[tokio::test] + async fn no_attachments_no_augmentation() { + let trace = LlmTrace::from_file(format!("{FIXTURES}/smoke_greeting.json")).unwrap(); + let rig = TestRigBuilder::new() + .with_trace(trace.clone()) + .build() + .await; + + rig.send_message("Hello! Introduce yourself briefly.").await; + let responses = rig.wait_for_responses(1, TIMEOUT).await; + + let requests = rig.captured_llm_requests(); + let last_request = &requests[requests.len() - 1]; + let last_user_msg = last_request + .iter() + .rev() + .find(|m| matches!(m.role, ironclaw::llm::Role::User)) + .expect("should have a user message"); + + // No attachments → no augmentation tags, no content parts + assert!( + !last_user_msg.content.contains(""), + "plain message should NOT contain " + ); + assert!( + last_user_msg.content_parts.is_empty(), + "plain message should have no content parts" + ); + + rig.verify_trace_expects(&trace, &responses); + rig.shutdown(); + } +} diff --git a/tests/e2e_routine_heartbeat.rs b/tests/e2e_routine_heartbeat.rs index 6f4dda34..1e65fb3d 100644 --- a/tests/e2e_routine_heartbeat.rs +++ b/tests/e2e_routine_heartbeat.rs @@ -203,6 +203,7 @@ mod tests { thread_id: None, received_at: Utc::now(), metadata: serde_json::json!({}), + attachments: Vec::new(), }; let fired = engine.check_event_triggers(&matching_msg).await; assert!( @@ -223,6 +224,7 @@ mod tests { thread_id: None, received_at: Utc::now(), metadata: serde_json::json!({}), + attachments: Vec::new(), }; let fired_neg = engine.check_event_triggers(&non_matching_msg).await; assert_eq!(fired_neg, 0, "Expected 0 routines fired on non-match"); @@ -286,6 +288,7 @@ mod tests { thread_id: None, received_at: Utc::now(), metadata: serde_json::json!({}), + attachments: Vec::new(), }; let fired1 = engine.check_event_triggers(&msg).await; assert!(fired1 >= 1, "First fire should work"); diff --git a/tests/fixtures/hello.pdf b/tests/fixtures/hello.pdf new file mode 100644 index 00000000..4214e98e --- /dev/null +++ b/tests/fixtures/hello.pdf @@ -0,0 +1,68 @@ +%PDF-1.3 +% ReportLab Generated PDF document (opensource) +1 0 obj +<< +/F1 2 0 R +>> +endobj +2 0 obj +<< +/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font +>> +endobj +3 0 obj +<< +/Contents 7 0 R /MediaBox [ 0 0 612 792 ] /Parent 6 0 R /Resources << +/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] +>> /Rotate 0 /Trans << + +>> + /Type /Page +>> +endobj +4 0 obj +<< +/PageMode /UseNone /Pages 6 0 R /Type /Catalog +>> +endobj +5 0 obj +<< +/Author (anonymous) /CreationDate (D:20260306140325-08'00') /Creator (anonymous) /Keywords () /ModDate (D:20260306140325-08'00') /Producer (ReportLab PDF Library - \(opensource\)) + /Subject (unspecified) /Title (untitled) /Trapped /False +>> +endobj +6 0 obj +<< +/Count 1 /Kids [ 3 0 R ] /Type /Pages +>> +endobj +7 0 obj +<< +/Filter [ /ASCII85Decode /FlateDecode ] /Length 102 +>> +stream +GapQh0E=F,0U\H3T\pNYT^QKk?tc>IP,;W#U1^23ihPEM_?CW4KISi90MjG.ifICK%?K#/S:$%[r1]\q9neZ[Kb,ht@Ke@a)FbAl~>endstream +endobj +xref +0 8 +0000000000 65535 f +0000000061 00000 n +0000000092 00000 n +0000000199 00000 n +0000000392 00000 n +0000000460 00000 n +0000000721 00000 n +0000000780 00000 n +trailer +<< +/ID +[<04d3222d792ab249042c58200a1c9b96><04d3222d792ab249042c58200a1c9b96>] +% ReportLab generated PDF document -- digest (opensource) + +/Info 5 0 R +/Root 4 0 R +/Size 8 +>> +startxref +972 +%%EOF diff --git a/tests/fixtures/llm_traces/spot/attachment_audio_transcript.json b/tests/fixtures/llm_traces/spot/attachment_audio_transcript.json new file mode 100644 index 00000000..2bb6fa73 --- /dev/null +++ b/tests/fixtures/llm_traces/spot/attachment_audio_transcript.json @@ -0,0 +1,21 @@ +{ + "model_name": "spot-attachment-audio-transcript", + "expects": { + "response_contains": ["transcript"], + "max_tool_calls": 0, + "min_responses": 1 + }, + "steps": [ + { + "request_hint": { + "last_user_message_contains": "" + }, + "response": { + "type": "text", + "content": "I can see the transcript from your audio attachment. You said: 'Hello, can you help me with my project?'. How can I help?", + "input_tokens": 80, + "output_tokens": 30 + } + } + ] +} diff --git a/tests/fixtures/llm_traces/spot/attachment_image.json b/tests/fixtures/llm_traces/spot/attachment_image.json new file mode 100644 index 00000000..557e0fb4 --- /dev/null +++ b/tests/fixtures/llm_traces/spot/attachment_image.json @@ -0,0 +1,21 @@ +{ + "model_name": "spot-attachment-image", + "expects": { + "response_contains": ["screenshot"], + "max_tool_calls": 0, + "min_responses": 1 + }, + "steps": [ + { + "request_hint": { + "last_user_message_contains": "sent as visual content" + }, + "response": { + "type": "text", + "content": "I can see the screenshot you shared. It appears to show a code editor with some Rust code. What would you like me to help with?", + "input_tokens": 200, + "output_tokens": 30 + } + } + ] +} diff --git a/tests/support/test_channel.rs b/tests/support/test_channel.rs index 09591c4f..12f45532 100644 --- a/tests/support/test_channel.rs +++ b/tests/support/test_channel.rs @@ -91,6 +91,11 @@ impl TestChannel { self.tx.send(msg).await.expect("TestChannel tx closed"); } + /// Inject a raw `IncomingMessage` (for tests that need attachments, etc.). + pub async fn send_incoming(&self, msg: IncomingMessage) { + self.tx.send(msg).await.expect("TestChannel tx closed"); + } + /// Inject a user message with a specific thread ID. pub async fn send_message_in_thread(&self, content: &str, thread_id: &str) { let msg = IncomingMessage::new("test", &self.user_id, content).with_thread(thread_id); diff --git a/tests/support/test_rig.rs b/tests/support/test_rig.rs index 430d9182..0073741e 100644 --- a/tests/support/test_rig.rs +++ b/tests/support/test_rig.rs @@ -131,6 +131,21 @@ impl TestRig { self.channel.send_message(content).await; } + /// Inject a raw `IncomingMessage` (for tests that need attachments, etc.). + pub async fn send_incoming(&self, msg: ironclaw::channels::IncomingMessage) { + self.channel.send_incoming(msg).await; + } + + /// Return all message lists that were sent to the LLM provider. + /// + /// Only available when the rig was built with a `TraceLlm` (i.e., via `.with_trace()`). + pub fn captured_llm_requests(&self) -> Vec> { + self.trace_llm + .as_ref() + .map(|t| t.captured_requests()) + .unwrap_or_default() + } + /// Wait until at least `n` responses have been captured, or `timeout` elapses. pub async fn wait_for_responses(&self, n: usize, timeout: Duration) -> Vec { self.channel.wait_for_responses(n, timeout).await @@ -607,6 +622,8 @@ impl TestRigBuilder { as Arc) } }, + transcription: None, + document_extraction: None, }; // 7. Create TestChannel and ChannelManager. diff --git a/tests/wit_compat.rs b/tests/wit_compat.rs index ad302b38..4dcacf4e 100644 --- a/tests/wit_compat.rs +++ b/tests/wit_compat.rs @@ -214,9 +214,9 @@ fn instantiate_tool_component( // If the WIT added/removed/renamed a function, stub registration // or instantiation will fail. - // Register stubs for both versioned (0.2.0+) and unversioned (pre-0.2.0) interface + // Register stubs for both versioned (0.3.0+) and unversioned (pre-0.3.0) interface // paths so that both old and new WASM artifacts can instantiate. - for interface in &["near:agent/host", "near:agent/host@0.2.0"] { + for interface in &["near:agent/host", "near:agent/host@0.3.0"] { let mut root = linker.root(); if let Ok(mut host) = root.instance(interface) { stub_shared_host_functions(&mut host)?; @@ -252,7 +252,7 @@ fn instantiate_channel_component( wasmtime_wasi::add_to_linker_sync(&mut linker) .map_err(|e| format!("WASI linker failed: {e}"))?; - // Register stubs for both versioned (0.2.0+) and unversioned (pre-0.2.0) interface + // Register stubs for both versioned (0.3.0+) and unversioned (pre-0.3.0) interface // paths so that both old and new WASM artifacts can instantiate. // Register stubs under both versioned and unversioned interface paths. // This helper avoids repeating the stub registration code. @@ -261,6 +261,12 @@ fn instantiate_channel_component( ) -> Result<(), String> { stub_shared_host_functions(host)?; + host.func_new("store-attachment-data", |_ctx, _args, results| { + results[0] = wasmtime::component::Val::Result(Ok(None)); + Ok(()) + }) + .map_err(|e| format!("stub 'store-attachment-data': {e}"))?; + host.func_new("emit-message", |_ctx, _args, _results| Ok(())) .map_err(|e| format!("stub 'emit-message': {e}"))?; @@ -307,8 +313,8 @@ fn instantiate_channel_component( { let mut root = linker.root(); let mut host = root - .instance("near:agent/channel-host@0.2.0") - .map_err(|e| format!("failed to create versioned channel-host: {e}"))?; + .instance("near:agent/channel-host@0.3.0") + .map_err(|e| format!("failed to create versioned channel-host@0.3.0: {e}"))?; stub_channel_host(&mut host)?; } @@ -505,7 +511,7 @@ fn wit_files_contain_version_annotation() { assert!( content.contains("package near:agent@"), - "{wit_file} must contain a versioned package declaration (e.g., 'package near:agent@0.2.0;')" + "{wit_file} must contain a versioned package declaration (e.g., 'package near:agent@0.3.0;')" ); } } diff --git a/tools-src/github/Cargo.toml b/tools-src/github/Cargo.toml index a9cc865d..7f1c2630 100644 --- a/tools-src/github/Cargo.toml +++ b/tools-src/github/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "github-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "GitHub integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" diff --git a/tools-src/github/github-tool.capabilities.json b/tools-src/github/github-tool.capabilities.json index bd92dcf5..48c53dbf 100644 --- a/tools-src/github/github-tool.capabilities.json +++ b/tools-src/github/github-tool.capabilities.json @@ -1,6 +1,6 @@ { - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "capabilities": { "http": { "allowlist": [ diff --git a/tools-src/gmail/Cargo.toml b/tools-src/gmail/Cargo.toml index 205292aa..1da6b4d7 100644 --- a/tools-src/gmail/Cargo.toml +++ b/tools-src/gmail/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "gmail-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Gmail integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" diff --git a/tools-src/gmail/gmail-tool.capabilities.json b/tools-src/gmail/gmail-tool.capabilities.json index 1ddafe7e..2e11d32b 100644 --- a/tools-src/gmail/gmail-tool.capabilities.json +++ b/tools-src/gmail/gmail-tool.capabilities.json @@ -1,6 +1,6 @@ { - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { diff --git a/tools-src/google-calendar/Cargo.toml b/tools-src/google-calendar/Cargo.toml index 0b5ef361..deef7e46 100644 --- a/tools-src/google-calendar/Cargo.toml +++ b/tools-src/google-calendar/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-calendar-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Google Calendar integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" diff --git a/tools-src/google-calendar/google-calendar-tool.capabilities.json b/tools-src/google-calendar/google-calendar-tool.capabilities.json index 86dd0c3c..15e756ae 100644 --- a/tools-src/google-calendar/google-calendar-tool.capabilities.json +++ b/tools-src/google-calendar/google-calendar-tool.capabilities.json @@ -1,6 +1,6 @@ { - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { diff --git a/tools-src/google-docs/Cargo.toml b/tools-src/google-docs/Cargo.toml index 8590c2be..c1142e6a 100644 --- a/tools-src/google-docs/Cargo.toml +++ b/tools-src/google-docs/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-docs-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Google Docs integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" diff --git a/tools-src/google-docs/google-docs-tool.capabilities.json b/tools-src/google-docs/google-docs-tool.capabilities.json index 386b0ba3..7a365c1d 100644 --- a/tools-src/google-docs/google-docs-tool.capabilities.json +++ b/tools-src/google-docs/google-docs-tool.capabilities.json @@ -1,6 +1,6 @@ { - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { diff --git a/tools-src/google-drive/Cargo.toml b/tools-src/google-drive/Cargo.toml index 3385c14a..7e9523b7 100644 --- a/tools-src/google-drive/Cargo.toml +++ b/tools-src/google-drive/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-drive-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Google Drive integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" diff --git a/tools-src/google-drive/google-drive-tool.capabilities.json b/tools-src/google-drive/google-drive-tool.capabilities.json index aa741fd6..53667933 100644 --- a/tools-src/google-drive/google-drive-tool.capabilities.json +++ b/tools-src/google-drive/google-drive-tool.capabilities.json @@ -1,6 +1,6 @@ { - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { diff --git a/tools-src/google-sheets/Cargo.toml b/tools-src/google-sheets/Cargo.toml index 048c44de..3ad44cd0 100644 --- a/tools-src/google-sheets/Cargo.toml +++ b/tools-src/google-sheets/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-sheets-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Google Sheets integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" diff --git a/tools-src/google-sheets/google-sheets-tool.capabilities.json b/tools-src/google-sheets/google-sheets-tool.capabilities.json index 97da6197..624c4381 100644 --- a/tools-src/google-sheets/google-sheets-tool.capabilities.json +++ b/tools-src/google-sheets/google-sheets-tool.capabilities.json @@ -1,6 +1,6 @@ { - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { diff --git a/tools-src/google-slides/Cargo.toml b/tools-src/google-slides/Cargo.toml index c0e3d42b..1eeed37d 100644 --- a/tools-src/google-slides/Cargo.toml +++ b/tools-src/google-slides/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "google-slides-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Google Slides integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" diff --git a/tools-src/google-slides/google-slides-tool.capabilities.json b/tools-src/google-slides/google-slides-tool.capabilities.json index 31e5c734..17334bc0 100644 --- a/tools-src/google-slides/google-slides-tool.capabilities.json +++ b/tools-src/google-slides/google-slides-tool.capabilities.json @@ -1,6 +1,6 @@ { - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { diff --git a/tools-src/slack/Cargo.toml b/tools-src/slack/Cargo.toml index ee22922c..2b11f560 100644 --- a/tools-src/slack/Cargo.toml +++ b/tools-src/slack/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "slack-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Slack integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" diff --git a/tools-src/slack/slack-tool.capabilities.json b/tools-src/slack/slack-tool.capabilities.json index 742e349a..8b9060d7 100644 --- a/tools-src/slack/slack-tool.capabilities.json +++ b/tools-src/slack/slack-tool.capabilities.json @@ -1,6 +1,6 @@ { - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { diff --git a/tools-src/telegram/Cargo.toml b/tools-src/telegram/Cargo.toml index 9af023c5..cdc2b3ec 100644 --- a/tools-src/telegram/Cargo.toml +++ b/tools-src/telegram/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "telegram-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Telegram user-mode integration tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" diff --git a/tools-src/telegram/telegram-tool.capabilities.json b/tools-src/telegram/telegram-tool.capabilities.json index cd42b5be..665baedd 100644 --- a/tools-src/telegram/telegram-tool.capabilities.json +++ b/tools-src/telegram/telegram-tool.capabilities.json @@ -1,6 +1,6 @@ { - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "http": { "allowlist": [ { diff --git a/tools-src/web-search/Cargo.toml b/tools-src/web-search/Cargo.toml index 9473883f..8bd29ff1 100644 --- a/tools-src/web-search/Cargo.toml +++ b/tools-src/web-search/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "web-search-tool" -version = "0.1.0" +version = "0.2.0" edition = "2021" description = "Brave Web Search tool for IronClaw (WASM component)" license = "MIT OR Apache-2.0" diff --git a/tools-src/web-search/web-search-tool.capabilities.json b/tools-src/web-search/web-search-tool.capabilities.json index 8ee5b4ac..bc660aaf 100644 --- a/tools-src/web-search/web-search-tool.capabilities.json +++ b/tools-src/web-search/web-search-tool.capabilities.json @@ -1,6 +1,6 @@ { - "version": "0.1.0", - "wit_version": "0.2.0", + "version": "0.2.0", + "wit_version": "0.3.0", "capabilities": { "http": { "allowlist": [ diff --git a/wit/channel.wit b/wit/channel.wit index f41db16d..c0eb4510 100644 --- a/wit/channel.wit +++ b/wit/channel.wit @@ -1,3 +1,5 @@ +package near:agent@0.3.0; + // WASM Channel Sandbox Interface // // Defines the contract between sandboxed channels and the host runtime. @@ -38,8 +40,6 @@ // - Workspace writes are prefixed with channels// to prevent escape // - Message emission is rate-limited -package near:agent@0.2.0; - /// Host-provided capabilities for sandboxed channels. /// /// Extends base tool capabilities with channel-specific functions: @@ -113,6 +113,50 @@ interface channel-host { // ==================== Channel-Specific Capabilities ==================== + /// A file or media attachment on an inbound message (channel → agent). + /// + /// Core fields are part of the record. Extended metadata (duration, dimensions, + /// codec, etc.) goes in `extras-json` to avoid WIT record changes when new + /// properties are needed. Binary data (e.g., downloaded voice bytes) should be + /// stored via `store-attachment-data` rather than inlined in the record. + record inbound-attachment { + /// Unique identifier within the channel (e.g., Telegram file_id). + id: string, + /// MIME type (e.g., "image/jpeg", "audio/ogg", "application/pdf"). + mime-type: string, + /// Original filename, if known. + filename: option, + /// File size in bytes, if known. + size-bytes: option, + /// URL to download the file from the channel's API. + /// May require authentication (handled by host credential injection). + source-url: option, + /// Opaque key for host-side storage (e.g., after download/caching). + storage-key: option, + /// Extracted text content (e.g., OCR result, PDF text, audio transcript). + extracted-text: option, + /// Extensible metadata as JSON string. + /// + /// Used for properties that may be added over time without changing WIT. + /// Well-known keys: + /// - "duration_secs": u32 — duration in seconds (audio/video) + /// - "width": u32, "height": u32 — pixel dimensions (images/video) + /// - "codec": string — audio/video codec + /// - "thumbnail_file_id": string — thumbnail identifier + extras-json: string, + } + + /// Store binary data for an attachment (e.g., downloaded voice note bytes). + /// + /// Call this before emit-message to associate raw bytes with an attachment. + /// The host retrieves the data after the callback using the attachment ID. + /// + /// Security: + /// - Maximum 20MB per attachment + /// - Maximum 50MB total per callback execution + /// - Data is cleared after the callback completes + store-attachment-data: func(attachment-id: string, data: list) -> result<_, string>; + /// A message to emit to the agent. record emitted-message { /// User identifier within the channel (e.g., Slack user ID). @@ -125,6 +169,8 @@ interface channel-host { thread-id: option, /// Channel-specific metadata as JSON string. metadata-json: string, + /// File or media attachments on this message. + attachments: list, } /// Emit a message to the agent. @@ -235,6 +281,18 @@ interface channel { body: list, } + /// A file or image attachment on an outbound message (agent → channel). + /// + /// Contains raw file bytes for the channel to upload/send. + record attachment { + /// Original filename (e.g., "screenshot.png"). + filename: string, + /// MIME type (e.g., "image/png"). + mime-type: string, + /// Raw file bytes. + data: list, + } + /// Agent response to be sent back to the channel. record agent-response { /// Unique message ID for correlation. @@ -245,6 +303,8 @@ interface channel { thread-id: option, /// Channel-specific metadata as JSON string. metadata-json: string, + /// File/image attachments to send. + attachments: list, } // ==================== Status Types ==================== @@ -340,6 +400,20 @@ interface channel { /// - update: The status update on-status: func(update: status-update); + /// Send a proactive message to a user without a prior incoming message. + /// + /// Used for broadcasts, alerts, and agent-initiated messages with attachments. + /// The user-id identifies the target user within the channel. + /// + /// Arguments: + /// - user-id: Target user identifier (e.g., Telegram chat_id) + /// - response: The message content and attachments to send + /// + /// Returns: + /// - Ok: Message delivered successfully + /// - Err(string): Delivery failure message + on-broadcast: func(user-id: string, response: agent-response) -> result<_, string>; + /// Clean up channel resources. /// /// Called when the channel is being unloaded. diff --git a/wit/tool.wit b/wit/tool.wit index aef3e22d..cfe2b591 100644 --- a/wit/tool.wit +++ b/wit/tool.wit @@ -1,3 +1,5 @@ +package near:agent@0.3.0; + // WASM Tool Sandbox Interface // // Defines the contract between sandboxed tools and the host runtime. @@ -9,8 +11,6 @@ // - Secrets are NEVER exposed to WASM; credentials are injected at host boundary // - All outputs are scanned for secret leakage before returning to WASM -package near:agent@0.2.0; - /// Host-provided capabilities for sandboxed tools. /// /// These are the only ways a sandboxed tool can interact with the outside world.