From bbb2d5c4ddb4482c2529962c831d1b57d0ecc7fb Mon Sep 17 00:00:00 2001 From: serrrfirat Date: Sat, 21 Feb 2026 14:56:08 +0400 Subject: [PATCH] feat: add audio transcription pipeline with OpenAI Whisper and Telegram voice notes (#90) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds speech-to-text support so WASM channels can emit audio attachments that get automatically transcribed before reaching the agent. Telegram voice notes are the first integration — downloaded via Bot API and transcribed via OpenAI Whisper. - Extend WIT with attachment-kind, attachment records on emitted-message - Add Attachment/AttachmentKind types to channel.rs and IncomingMessage - Add TranscriptionProvider trait, AudioFormat enum, TranscriptionMiddleware - Implement OpenAI Whisper provider (multipart POST, 25MB limit) - Add TranscriptionConfig + TranscriptionSettings with env var overrides - Parse Telegram voice messages, download via getFile, emit as attachments - Apply transcription in both process/dispatch emitted message paths - Graceful degradation: download failures show "[Voice note: download failed]" - Validate attachment sizes (10MB max), drop oversized without losing message Co-Authored-By: Claude Opus 4.6 --- Cargo.toml | 2 +- channels-src/telegram/Cargo.toml | 2 - channels-src/telegram/src/lib.rs | 159 +++++++- .../telegram/telegram.capabilities.json | 2 +- src/channels/channel.rs | 35 ++ src/channels/mod.rs | 5 +- src/channels/wasm/host.rs | 32 +- src/channels/wasm/wrapper.rs | 76 +++- src/config/mod.rs | 4 + src/config/transcription.rs | 195 +++++++++ src/lib.rs | 1 + src/main.rs | 32 +- src/settings.rs | 44 ++ src/transcription/mod.rs | 375 ++++++++++++++++++ src/transcription/openai.rs | 196 +++++++++ wit/channel.wit | 23 ++ 16 files changed, 1167 insertions(+), 16 deletions(-) create mode 100644 src/config/transcription.rs create mode 100644 src/transcription/mod.rs create mode 100644 src/transcription/openai.rs diff --git a/Cargo.toml b/Cargo.toml index 3bd8e947..4cd6f663 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,7 +41,7 @@ tokio-stream = { version = "0.1", features = ["sync"] } futures = "0.3" # HTTP client -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots", "stream"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls-native-roots", "stream"] } # Serialization serde = { version = "1", features = ["derive"] } diff --git a/channels-src/telegram/Cargo.toml b/channels-src/telegram/Cargo.toml index 06cd9de5..1964e327 100644 --- a/channels-src/telegram/Cargo.toml +++ b/channels-src/telegram/Cargo.toml @@ -25,5 +25,3 @@ opt-level = "s" lto = true strip = true codegen-units = 1 - -[workspace] diff --git a/channels-src/telegram/src/lib.rs b/channels-src/telegram/src/lib.rs index 5c2f91af..b4ffe038 100644 --- a/channels-src/telegram/src/lib.rs +++ b/channels-src/telegram/src/lib.rs @@ -76,6 +76,9 @@ struct TelegramMessage { #[serde(default)] caption: Option, + /// Voice message. + voice: Option, + /// Original message if this is a reply. reply_to_message: Option>, @@ -139,6 +142,36 @@ struct MessageEntity { user: Option, } +/// Telegram Voice object. +/// https://core.telegram.org/bots/api#voice +#[derive(Debug, Deserialize)] +struct TelegramVoice { + /// Identifier for this file, which can be used to download the file. + file_id: String, + + /// Duration of the audio in seconds. + duration: u32, + + /// MIME type of the file. + #[serde(default)] + mime_type: Option, + + /// File size in bytes. + #[serde(default)] + file_size: Option, +} + +/// Telegram File object returned by getFile. +/// https://core.telegram.org/bots/api#file +#[derive(Debug, Deserialize)] +struct TelegramFile { + /// Identifier for this file. + file_id: String, + + /// File path for downloading. Use https://api.telegram.org/file/bot/. + file_path: Option, +} + /// Telegram API response wrapper. #[derive(Debug, Deserialize)] struct TelegramApiResponse { @@ -867,6 +900,75 @@ fn send_pairing_reply(chat_id: i64, code: &str) -> Result<(), String> { } } +// ============================================================================ +// Voice File Download +// ============================================================================ + +/// Download a voice file from Telegram by file_id. +/// +/// 1. Call getFile to get the file_path. +/// 2. Download the file bytes from /file/bot{TOKEN}/{file_path}. +fn download_voice_file(file_id: &str) -> Result, String> { + // Step 1: Call getFile to get file_path + // Double braces `{{...}}` produce a literal `{TELEGRAM_BOT_TOKEN}` placeholder + // in the URL, which the host-side credential injector replaces with the real token. + let get_file_url = format!( + "https://api.telegram.org/bot{{TELEGRAM_BOT_TOKEN}}/getFile?file_id={}", + file_id + ); + + let headers = serde_json::json!({}); + let result = channel_host::http_request("GET", &get_file_url, &headers.to_string(), None, None); + + let response = result.map_err(|e| format!("getFile request failed: {}", e))?; + + if response.status != 200 { + let body_str = String::from_utf8_lossy(&response.body); + return Err(format!("getFile returned {}: {}", response.status, body_str)); + } + + let api_response: TelegramApiResponse = + serde_json::from_slice(&response.body) + .map_err(|e| format!("Failed to parse getFile response: {}", e))?; + + if !api_response.ok { + return Err(format!( + "getFile API error: {}", + api_response + .description + .unwrap_or_else(|| "unknown".to_string()) + )); + } + + let file = api_response + .result + .ok_or_else(|| "getFile returned no result".to_string())?; + + let file_path = file + .file_path + .ok_or_else(|| "getFile returned no file_path".to_string())?; + + // Step 2: Download the actual file bytes + let download_url = format!( + "https://api.telegram.org/file/bot{{TELEGRAM_BOT_TOKEN}}/{}", + file_path + ); + + let result = + channel_host::http_request("GET", &download_url, &headers.to_string(), None, None); + + let response = result.map_err(|e| format!("File download failed: {}", e))?; + + if response.status != 200 { + return Err(format!( + "File download returned status {}", + response.status + )); + } + + Ok(response.body) +} + // ============================================================================ // Update Handling // ============================================================================ @@ -886,6 +988,9 @@ fn handle_update(update: TelegramUpdate) { /// Process a single message. fn handle_message(message: TelegramMessage) { + // Check for voice note first (voice-only messages have no text) + let is_voice = message.voice.is_some(); + // Use text or caption (for media messages) let content = message .text @@ -893,7 +998,8 @@ fn handle_message(message: TelegramMessage) { .or_else(|| message.caption.filter(|c| !c.is_empty())) .unwrap_or_default(); - if content.is_empty() { + // Allow voice notes through even when content is empty + if content.is_empty() && !is_voice { return; } @@ -1038,19 +1144,65 @@ fn handle_message(message: TelegramMessage) { }, ); + // Handle voice notes: download and attach audio bytes. + // Note: download is synchronous (two HTTP roundtrips to Telegram API). + // This blocks the WASM execution for the current polling tick. + let mut attachments = Vec::new(); + let mut voice_download_failed = false; + if let Some(ref voice) = message.voice { + channel_host::log( + channel_host::LogLevel::Info, + &format!( + "Voice note from user {} (duration: {}s, file_id: {})", + from.id, voice.duration, voice.file_id + ), + ); + + match download_voice_file(&voice.file_id) { + Ok(audio_bytes) => { + channel_host::log( + channel_host::LogLevel::Info, + &format!("Downloaded voice file: {} bytes", audio_bytes.len()), + ); + attachments.push(channel_host::Attachment { + kind: channel_host::AttachmentKind::Audio, + mime_type: voice + .mime_type + .clone() + .unwrap_or_else(|| "audio/ogg".to_string()), + data: audio_bytes, + filename: Some(format!("voice_{}.ogg", voice.file_id)), + duration_secs: Some(voice.duration), + }); + } + Err(e) => { + channel_host::log( + channel_host::LogLevel::Error, + &format!("Failed to download voice file: {}", e), + ); + voice_download_failed = true; + } + } + } + // Determine what to emit to the agent. + // - Voice notes: use "[Voice note]" as content (transcription happens host-side) // - `/start` (no args): emit a welcome placeholder so the agent greets the user // - Other bare `/commands` (e.g. /interrupt, /help): pass the raw command through // so Submission::parse() can handle it // - Commands with args (e.g. `/start hello`): cleaned_text already has the args // - Plain text: pass through as-is let trimmed_content = content.trim(); - let content_to_emit = if trimmed_content.eq_ignore_ascii_case("/start") { + let content_to_emit = if is_voice && voice_download_failed && content.is_empty() { + "[Voice note: download failed]".to_string() + } else if is_voice && content.is_empty() { + "[Voice note]".to_string() + } else if trimmed_content.eq_ignore_ascii_case("/start") { "[User started the bot]".to_string() } else if cleaned_text.is_empty() && trimmed_content.starts_with('/') { // Bare control command like /interrupt, /stop, /help — pass through raw trimmed_content.to_string() - } else if cleaned_text.is_empty() { + } else if cleaned_text.is_empty() && !is_voice { return; } else { cleaned_text @@ -1063,6 +1215,7 @@ fn handle_message(message: TelegramMessage) { content: content_to_emit, thread_id: None, // Telegram doesn't have threads in the same way metadata_json, + attachments, }); channel_host::log( diff --git a/channels-src/telegram/telegram.capabilities.json b/channels-src/telegram/telegram.capabilities.json index 41735b52..ede4bdcc 100644 --- a/channels-src/telegram/telegram.capabilities.json +++ b/channels-src/telegram/telegram.capabilities.json @@ -1 +1 @@ -{"type":"channel","name":"telegram","description":"Telegram Bot API channel for receiving and responding to Telegram messages","capabilities":{"http":{"allowlist":[{"host":"api.telegram.org","path_prefix":"/bot"}],"credentials":{"telegram_bot":{"secret_name":"telegram_bot_token","location":{"type":"url_path","placeholder":"{TELEGRAM_BOT_TOKEN}"},"host_patterns":["api.telegram.org"]}},"rate_limit":{"requests_per_minute":30,"requests_per_hour":1000}},"secrets":{"allowed_names":["telegram_*"]},"channel":{"allowed_paths":["/webhook/telegram"],"allow_polling":true,"min_poll_interval_ms":30000,"workspace_prefix":"channels/telegram/","emit_rate_limit":{"messages_per_minute":100,"messages_per_hour":5000}}},"config":{"bot_username":null,"owner_id":null,"respond_to_all_group_messages":false,"polling_enabled":false,"poll_interval_ms":30000,"dm_policy":"pairing","allow_from":[]}} +{"type":"channel","name":"telegram","description":"Telegram Bot API channel for receiving and responding to Telegram messages","capabilities":{"http":{"allowlist":[{"host":"api.telegram.org","path_prefix":"/bot"},{"host":"api.telegram.org","path_prefix":"/file/bot"}],"credentials":{"telegram_bot":{"secret_name":"telegram_bot_token","location":{"type":"url_path","placeholder":"{TELEGRAM_BOT_TOKEN}"},"host_patterns":["api.telegram.org"]}},"rate_limit":{"requests_per_minute":30,"requests_per_hour":1000}},"secrets":{"allowed_names":["telegram_*"]},"channel":{"allowed_paths":["/webhook/telegram"],"allow_polling":true,"min_poll_interval_ms":30000,"workspace_prefix":"channels/telegram/","emit_rate_limit":{"messages_per_minute":100,"messages_per_hour":5000}}},"config":{"bot_username":null,"owner_id":null,"respond_to_all_group_messages":false,"polling_enabled":false,"poll_interval_ms":30000,"dm_policy":"pairing","allow_from":[]}} diff --git a/src/channels/channel.rs b/src/channels/channel.rs index d87c8240..d6031339 100644 --- a/src/channels/channel.rs +++ b/src/channels/channel.rs @@ -9,6 +9,32 @@ use uuid::Uuid; use crate::error::ChannelError; +/// Kind of attachment carried on an incoming message. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AttachmentKind { + /// Audio content (voice notes, audio files). + Audio, + /// Image content (photos, screenshots). + Image, + /// Document content (PDFs, files). + Document, +} + +/// Binary attachment on a message (e.g., voice note, photo). +#[derive(Debug, Clone)] +pub struct Attachment { + /// What kind of content this is. + pub kind: AttachmentKind, + /// MIME type (e.g., "audio/ogg", "image/jpeg"). + pub mime_type: String, + /// Raw bytes of the attachment. + pub data: Vec, + /// Optional filename. + pub filename: Option, + /// Duration in seconds (for audio/video). + pub duration_secs: Option, +} + /// A message received from an external channel. #[derive(Debug, Clone)] pub struct IncomingMessage { @@ -28,6 +54,8 @@ pub struct IncomingMessage { pub received_at: DateTime, /// Channel-specific metadata. pub metadata: serde_json::Value, + /// Binary attachments (voice notes, images, etc.). + pub attachments: Vec, } impl IncomingMessage { @@ -46,6 +74,7 @@ impl IncomingMessage { thread_id: None, received_at: Utc::now(), metadata: serde_json::Value::Null, + attachments: Vec::new(), } } @@ -66,6 +95,12 @@ impl IncomingMessage { self.user_name = Some(name.into()); self } + + /// Set attachments. + pub fn with_attachments(mut self, attachments: Vec) -> Self { + self.attachments = attachments; + self + } } /// Stream of incoming messages. diff --git a/src/channels/mod.rs b/src/channels/mod.rs index 08d742e8..d6d855ac 100644 --- a/src/channels/mod.rs +++ b/src/channels/mod.rs @@ -35,7 +35,10 @@ pub mod wasm; pub mod web; mod webhook_server; -pub use channel::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate}; +pub use channel::{ + Attachment, AttachmentKind, Channel, IncomingMessage, MessageStream, OutgoingResponse, + StatusUpdate, +}; pub use http::HttpChannel; pub use manager::ChannelManager; pub use repl::ReplChannel; diff --git a/src/channels/wasm/host.rs b/src/channels/wasm/host.rs index 03a6170f..32e358d6 100644 --- a/src/channels/wasm/host.rs +++ b/src/channels/wasm/host.rs @@ -7,6 +7,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; +use crate::channels::channel::Attachment; use crate::channels::wasm::capabilities::{ChannelCapabilities, EmitRateLimitConfig}; use crate::channels::wasm::error::WasmChannelError; use crate::tools::wasm::{HostState, LogLevel}; @@ -17,6 +18,9 @@ const MAX_EMITS_PER_EXECUTION: usize = 100; /// Maximum message content size (64 KB). const MAX_MESSAGE_CONTENT_SIZE: usize = 64 * 1024; +/// Maximum size for a single attachment (10 MB). +const MAX_ATTACHMENT_SIZE: usize = 10 * 1024 * 1024; + /// A message emitted by a WASM channel to be sent to the agent. #[derive(Debug, Clone)] pub struct EmittedMessage { @@ -37,6 +41,9 @@ pub struct EmittedMessage { /// Timestamp when the message was emitted. pub emitted_at_millis: u64, + + /// Binary attachments (voice notes, images, etc.). + pub attachments: Vec, } impl EmittedMessage { @@ -52,6 +59,7 @@ impl EmittedMessage { .duration_since(UNIX_EPOCH) .map(|d| d.as_millis() as u64) .unwrap_or(0), + attachments: Vec::new(), } } @@ -72,6 +80,12 @@ impl EmittedMessage { self.metadata_json = metadata_json.into(); self } + + /// Set attachments. + pub fn with_attachments(mut self, attachments: Vec) -> Self { + self.attachments = attachments; + self + } } /// A pending workspace write operation. @@ -168,7 +182,7 @@ impl ChannelHostState { /// /// Messages are queued and delivered after callback execution completes. /// Rate limiting is enforced per-execution and globally. - pub fn emit_message(&mut self, msg: EmittedMessage) -> Result<(), WasmChannelError> { + pub fn emit_message(&mut self, mut msg: EmittedMessage) -> Result<(), WasmChannelError> { // Check per-execution limit if !self.emit_enabled { self.emits_dropped += 1; @@ -186,6 +200,22 @@ impl ChannelHostState { return Ok(()); } + // Validate attachment sizes — drop only oversized attachments, not the whole message + msg.attachments.retain(|attachment| { + if attachment.data.len() > MAX_ATTACHMENT_SIZE { + tracing::warn!( + channel = %self.channel_name, + size = attachment.data.len(), + max = MAX_ATTACHMENT_SIZE, + mime = %attachment.mime_type, + "Attachment too large, dropping attachment (message still delivered)" + ); + false + } else { + true + } + }); + // Validate message content size if msg.content.len() > MAX_MESSAGE_CONTENT_SIZE { tracing::warn!( diff --git a/src/channels/wasm/wrapper.rs b/src/channels/wasm/wrapper.rs index f546da85..10c5ca94 100644 --- a/src/channels/wasm/wrapper.rs +++ b/src/channels/wasm/wrapper.rs @@ -40,6 +40,7 @@ use wasmtime::Store; use wasmtime::component::Linker; use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView}; +use crate::channels::channel::{Attachment, AttachmentKind}; use crate::channels::wasm::capabilities::ChannelCapabilities; use crate::channels::wasm::error::WasmChannelError; use crate::channels::wasm::host::{ @@ -436,6 +437,7 @@ impl near::agent::channel_host::Host for ChannelStoreData { user_id = %msg.user_id, user_name = ?msg.user_name, content_len = msg.content.len(), + attachment_count = msg.attachments.len(), "WASM emit_message called" ); @@ -448,6 +450,31 @@ impl near::agent::channel_host::Host for ChannelStoreData { } emitted = emitted.with_metadata(msg.metadata_json); + // Convert WIT attachments to Rust types + if !msg.attachments.is_empty() { + let attachments = msg + .attachments + .into_iter() + .map(|a| { + let kind = match a.kind { + near::agent::channel_host::AttachmentKind::Audio => AttachmentKind::Audio, + near::agent::channel_host::AttachmentKind::Image => AttachmentKind::Image, + near::agent::channel_host::AttachmentKind::Document => { + AttachmentKind::Document + } + }; + Attachment { + kind, + mime_type: a.mime_type, + data: a.data, + filename: a.filename, + duration_secs: a.duration_secs, + } + }) + .collect(); + emitted = emitted.with_attachments(attachments); + } + match self.host_state.emit_message(emitted) { Ok(()) => { tracing::info!("Message emitted to host state successfully"); @@ -553,6 +580,9 @@ pub struct WasmChannel { /// In-memory workspace store persisting writes across callback invocations. /// Ensures WASM channels can maintain state (e.g., polling offsets) between ticks. workspace_store: Arc, + + /// Optional transcription middleware for audio attachments. + transcription_middleware: Option>, } impl WasmChannel { @@ -584,9 +614,18 @@ impl WasmChannel { typing_task: RwLock::new(None), pairing_store, workspace_store: Arc::new(ChannelWorkspaceStore::new()), + transcription_middleware: None, } } + /// Set the transcription middleware for audio attachment processing. + pub fn set_transcription_middleware( + &mut self, + middleware: Arc, + ) { + self.transcription_middleware = Some(middleware); + } + /// Update the channel config before starting. /// /// Merges the provided values into the existing config JSON. @@ -1510,11 +1549,21 @@ impl WasmChannel { msg = msg.with_metadata(metadata); } - // Send to stream + // Carry attachments through (moves emitted.attachments into msg) + if !emitted.attachments.is_empty() { + msg = msg.with_attachments(emitted.attachments); + } + + // Apply transcription middleware if available (may replace content with transcript) + if let Some(ref middleware) = self.transcription_middleware { + msg = middleware.process(msg).await; + } + + // Send to stream (log post-transcription state intentionally) tracing::info!( channel = %self.name, - user_id = %emitted.user_id, - content_len = emitted.content.len(), + user_id = %msg.user_id, + content_len = msg.content.len(), "Sending emitted message to agent" ); @@ -1551,6 +1600,7 @@ impl WasmChannel { let pairing_store = self.pairing_store.clone(); let callback_timeout = self.runtime.config().callback_timeout; let workspace_store = self.workspace_store.clone(); + let transcription_middleware = self.transcription_middleware.clone(); tokio::spawn(async move { let mut interval_timer = tokio::time::interval(interval); @@ -1585,6 +1635,7 @@ impl WasmChannel { emitted_messages, &message_tx, &rate_limiter, + transcription_middleware.as_deref(), ).await { tracing::warn!( channel = %channel_name, @@ -1708,6 +1759,7 @@ impl WasmChannel { messages: Vec, message_tx: &RwLock>>, rate_limiter: &RwLock, + transcription_middleware: Option<&crate::transcription::TranscriptionMiddleware>, ) -> Result<(), WasmChannelError> { tracing::info!( channel = %channel_name, @@ -1755,11 +1807,21 @@ impl WasmChannel { msg = msg.with_metadata(metadata); } - // Send to stream + // Carry attachments through (moves emitted.attachments into msg) + if !emitted.attachments.is_empty() { + msg = msg.with_attachments(emitted.attachments); + } + + // Apply transcription middleware if available (may replace content with transcript) + if let Some(middleware) = transcription_middleware { + msg = middleware.process(msg).await; + } + + // Send to stream (log post-transcription state intentionally) tracing::info!( channel = %channel_name, - user_id = %emitted.user_id, - content_len = emitted.content.len(), + user_id = %msg.user_id, + content_len = msg.content.len(), "Sending polled message to agent" ); @@ -2332,6 +2394,7 @@ mod tests { messages, &message_tx, &rate_limiter, + None, ) .await; @@ -2370,6 +2433,7 @@ mod tests { messages, &message_tx, &rate_limiter, + None, ) .await; diff --git a/src/config/mod.rs b/src/config/mod.rs index 9326b682..cad40b24 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -19,6 +19,7 @@ mod safety; mod sandbox; mod secrets; mod skills; +mod transcription; mod tunnel; mod wasm; @@ -45,6 +46,7 @@ pub use self::safety::SafetyConfig; pub use self::sandbox::{ClaudeCodeConfig, SandboxModeConfig}; pub use self::secrets::SecretsConfig; pub use self::skills::SkillsConfig; +pub use self::transcription::TranscriptionConfig; pub use self::tunnel::TunnelConfig; pub use self::wasm::WasmConfig; @@ -74,6 +76,7 @@ pub struct Config { pub sandbox: SandboxModeConfig, pub claude_code: ClaudeCodeConfig, pub skills: SkillsConfig, + pub transcription: TranscriptionConfig, pub observability: crate::observability::ObservabilityConfig, } @@ -198,6 +201,7 @@ impl Config { sandbox: SandboxModeConfig::resolve()?, claude_code: ClaudeCodeConfig::resolve()?, skills: SkillsConfig::resolve()?, + transcription: TranscriptionConfig::resolve(settings)?, observability: crate::observability::ObservabilityConfig { backend: std::env::var("OBSERVABILITY_BACKEND").unwrap_or_else(|_| "none".into()), }, diff --git a/src/config/transcription.rs b/src/config/transcription.rs new file mode 100644 index 00000000..99e68633 --- /dev/null +++ b/src/config/transcription.rs @@ -0,0 +1,195 @@ +use secrecy::SecretString; + +use crate::config::helpers::optional_env; +use crate::error::ConfigError; +use crate::settings::Settings; + +/// Transcription provider configuration. +#[derive(Debug, Clone)] +pub struct TranscriptionConfig { + /// Whether transcription is enabled. + pub enabled: bool, + /// Provider to use: "openai". + pub provider: String, + /// OpenAI API key (reused from embeddings/LLM config). + pub openai_api_key: Option, + /// Model to use for transcription. + pub model: String, + /// Optional language hint (ISO-639-1, e.g., "en"). + pub language: Option, +} + +impl Default for TranscriptionConfig { + fn default() -> Self { + Self { + enabled: false, + provider: "openai".to_string(), + openai_api_key: None, + model: "whisper-1".to_string(), + language: None, + } + } +} + +impl TranscriptionConfig { + pub(crate) fn resolve(settings: &Settings) -> Result { + let openai_api_key = optional_env("OPENAI_API_KEY")?.map(SecretString::from); + + let provider = optional_env("TRANSCRIPTION_PROVIDER")? + .unwrap_or_else(|| settings.transcription.provider.clone()); + + let model = optional_env("TRANSCRIPTION_MODEL")? + .unwrap_or_else(|| settings.transcription.model.clone()); + + let language = optional_env("TRANSCRIPTION_LANGUAGE")? + .or_else(|| settings.transcription.language.clone()); + + let enabled = optional_env("TRANSCRIPTION_ENABLED")? + .map(|s| s.parse()) + .transpose() + .map_err(|e| ConfigError::InvalidValue { + key: "TRANSCRIPTION_ENABLED".to_string(), + message: format!("must be 'true' or 'false': {e}"), + })? + .unwrap_or(settings.transcription.enabled); + + // Only "openai" is currently supported + if enabled && provider != "openai" { + return Err(ConfigError::InvalidValue { + key: "TRANSCRIPTION_PROVIDER".to_string(), + message: format!( + "unsupported provider '{}', only 'openai' is currently supported", + provider + ), + }); + } + + Ok(Self { + enabled, + provider, + openai_api_key, + model, + language, + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::helpers::ENV_MUTEX; + use crate::settings::{Settings, TranscriptionSettings}; + + fn clear_transcription_env() { + unsafe { + std::env::remove_var("TRANSCRIPTION_ENABLED"); + std::env::remove_var("TRANSCRIPTION_PROVIDER"); + std::env::remove_var("TRANSCRIPTION_MODEL"); + std::env::remove_var("TRANSCRIPTION_LANGUAGE"); + std::env::remove_var("OPENAI_API_KEY"); + } + } + + #[test] + fn transcription_defaults_from_settings() { + let _guard = ENV_MUTEX.lock().expect("env mutex poisoned"); + clear_transcription_env(); + + let settings = Settings::default(); + let config = TranscriptionConfig::resolve(&settings).expect("resolve should succeed"); + + assert!(!config.enabled); + assert_eq!(config.provider, "openai"); + assert_eq!(config.model, "whisper-1"); + assert!(config.language.is_none()); + } + + #[test] + fn transcription_env_overrides_settings() { + let _guard = ENV_MUTEX.lock().expect("env mutex poisoned"); + clear_transcription_env(); + + unsafe { + std::env::set_var("TRANSCRIPTION_ENABLED", "true"); + std::env::set_var("TRANSCRIPTION_MODEL", "whisper-large-v3"); + std::env::set_var("TRANSCRIPTION_LANGUAGE", "en"); + } + + let settings = Settings::default(); + let config = TranscriptionConfig::resolve(&settings).expect("resolve should succeed"); + + assert!(config.enabled); + assert_eq!(config.model, "whisper-large-v3"); + assert_eq!(config.language, Some("en".to_string())); + + unsafe { + std::env::remove_var("TRANSCRIPTION_ENABLED"); + std::env::remove_var("TRANSCRIPTION_MODEL"); + std::env::remove_var("TRANSCRIPTION_LANGUAGE"); + } + } + + #[test] + fn transcription_settings_with_custom_values() { + let _guard = ENV_MUTEX.lock().expect("env mutex poisoned"); + clear_transcription_env(); + + let settings = Settings { + transcription: TranscriptionSettings { + enabled: true, + model: "whisper-large-v3".to_string(), + language: Some("fr".to_string()), + ..Default::default() + }, + ..Default::default() + }; + + let config = TranscriptionConfig::resolve(&settings).expect("resolve should succeed"); + + assert!(config.enabled); + assert_eq!(config.model, "whisper-large-v3"); + assert_eq!(config.language, Some("fr".to_string())); + } + + #[test] + fn transcription_rejects_unsupported_provider() { + let _guard = ENV_MUTEX.lock().expect("env mutex poisoned"); + clear_transcription_env(); + + let settings = Settings { + transcription: TranscriptionSettings { + enabled: true, + provider: "deepgram".to_string(), + ..Default::default() + }, + ..Default::default() + }; + + let result = TranscriptionConfig::resolve(&settings); + assert!(result.is_err()); + let err_msg = result.unwrap_err().to_string(); + assert!( + err_msg.contains("unsupported provider"), + "Error should mention unsupported provider, got: {err_msg}" + ); + } + + #[test] + fn transcription_disabled_skips_provider_validation() { + let _guard = ENV_MUTEX.lock().expect("env mutex poisoned"); + clear_transcription_env(); + + // When disabled, any provider string is accepted (never used) + let settings = Settings { + transcription: TranscriptionSettings { + enabled: false, + provider: "nonexistent".to_string(), + ..Default::default() + }, + ..Default::default() + }; + + let config = TranscriptionConfig::resolve(&settings).expect("should succeed when disabled"); + assert!(!config.enabled); + } +} diff --git a/src/lib.rs b/src/lib.rs index d14d14d2..0f640702 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -67,6 +67,7 @@ pub mod setup; pub mod skills; pub mod tools; pub mod tracing_fmt; +pub mod transcription; pub mod tunnel; pub mod util; pub mod worker; diff --git a/src/main.rs b/src/main.rs index f02da188..e066728f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1028,6 +1028,32 @@ async fn main() -> anyhow::Result<()> { // Collect webhook route fragments; a single WebhookServer hosts them all. let mut webhook_routes: Vec = Vec::new(); + // Build transcription middleware if enabled. + let transcription_middleware: Option> = + if config.transcription.enabled { + if let Some(ref api_key) = config.transcription.openai_api_key { + let provider = Arc::new(ironclaw::transcription::openai::OpenAiWhisper::new( + api_key.clone(), + config.transcription.model.clone(), + )); + let middleware = ironclaw::transcription::TranscriptionMiddleware::new( + provider, + config.transcription.language.clone(), + ); + tracing::info!( + provider = %config.transcription.provider, + model = %config.transcription.model, + "Audio transcription enabled" + ); + Some(Arc::new(middleware)) + } else { + tracing::warn!("Transcription enabled but OPENAI_API_KEY not set, disabling"); + None + } + } else { + None + }; + // Load WASM channels and register their webhook routes. if config.channels.wasm_channels_enabled && config.channels.wasm_channels_dir.exists() { match WasmChannelRuntime::new(WasmChannelRuntimeConfig::default()) { @@ -1072,7 +1098,11 @@ async fn main() -> anyhow::Result<()> { require_secret: webhook_secret.is_some(), }]; - let channel_arc = Arc::new(loaded.channel); + let mut channel = loaded.channel; + if let Some(ref mw) = transcription_middleware { + channel.set_transcription_middleware(Arc::clone(mw)); + } + let channel_arc = Arc::new(channel); { let mut config_updates = std::collections::HashMap::new(); diff --git a/src/settings.rs b/src/settings.rs index b1b12347..92e0bd0b 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -63,6 +63,11 @@ pub struct Settings { #[serde(default)] pub embeddings: EmbeddingsSettings, + // === Transcription (STT) === + /// Transcription configuration for voice notes. + #[serde(default)] + pub transcription: TranscriptionSettings, + // === Step 6: Channels === /// Tunnel configuration for public webhook endpoints. #[serde(default)] @@ -146,6 +151,45 @@ impl Default for EmbeddingsSettings { } } +/// Transcription (STT) configuration for voice notes. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TranscriptionSettings { + /// Whether transcription is enabled. + #[serde(default)] + pub enabled: bool, + + /// Provider to use: "openai". + #[serde(default = "default_transcription_provider")] + pub provider: String, + + /// Model to use for transcription. + #[serde(default = "default_transcription_model")] + pub model: String, + + /// Optional language hint (ISO-639-1, e.g., "en"). + #[serde(default)] + pub language: Option, +} + +fn default_transcription_provider() -> String { + "openai".to_string() +} + +fn default_transcription_model() -> String { + "whisper-1".to_string() +} + +impl Default for TranscriptionSettings { + fn default() -> Self { + Self { + enabled: false, + provider: default_transcription_provider(), + model: default_transcription_model(), + language: None, + } + } +} + /// Tunnel settings for public webhook endpoints. /// /// The tunnel URL is shared across all channels that need webhooks. diff --git a/src/transcription/mod.rs b/src/transcription/mod.rs new file mode 100644 index 00000000..41305d4d --- /dev/null +++ b/src/transcription/mod.rs @@ -0,0 +1,375 @@ +//! Audio transcription for voice notes and audio attachments. +//! +//! Provides a `TranscriptionProvider` trait and middleware that automatically +//! transcribes audio attachments on incoming messages before they reach the agent. +//! +//! # Architecture +//! +//! ```text +//! [WASM Channel] → emit-message { attachments=[audio bytes] } +//! → [Host: EmittedMessage → IncomingMessage] +//! → [TranscriptionMiddleware: detect audio, call provider, replace content] +//! → [Agent Loop: sees plain text] +//! ``` + +pub mod openai; + +use std::sync::Arc; + +use async_trait::async_trait; + +use crate::channels::{AttachmentKind, IncomingMessage}; + +/// Supported audio formats for transcription. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AudioFormat { + /// OGG with Opus codec (Telegram voice notes). + OggOpus, + /// MP3. + Mp3, + /// WAV. + Wav, + /// WebM. + Webm, + /// M4A / AAC. + M4a, +} + +impl AudioFormat { + /// File extension for this format. + pub fn extension(&self) -> &'static str { + match self { + Self::OggOpus => "ogg", + Self::Mp3 => "mp3", + Self::Wav => "wav", + Self::Webm => "webm", + Self::M4a => "m4a", + } + } + + /// MIME type for this format. + pub fn mime_type(&self) -> &'static str { + match self { + Self::OggOpus => "audio/ogg", + Self::Mp3 => "audio/mpeg", + Self::Wav => "audio/wav", + Self::Webm => "audio/webm", + Self::M4a => "audio/mp4", + } + } + + /// Detect format from MIME type string. + pub fn from_mime_type(mime: &str) -> Option { + // Normalize: strip parameters (e.g., "audio/ogg; codecs=opus" → "audio/ogg") + let base = mime.split(';').next().unwrap_or(mime).trim(); + match base { + "audio/ogg" | "audio/opus" => Some(Self::OggOpus), + "audio/mpeg" | "audio/mp3" => Some(Self::Mp3), + "audio/wav" | "audio/x-wav" | "audio/wave" => Some(Self::Wav), + "audio/webm" => Some(Self::Webm), + "audio/mp4" | "audio/m4a" | "audio/x-m4a" | "audio/aac" => Some(Self::M4a), + _ => None, + } + } +} + +/// Errors from transcription operations. +#[derive(Debug, thiserror::Error)] +pub enum TranscriptionError { + /// Unsupported audio format. + #[error("unsupported audio format: {mime_type}")] + UnsupportedFormat { mime_type: String }, + + /// Audio file exceeds the provider's size limit. + #[error("audio file too large: {size} bytes (max {max})")] + FileTooLarge { size: usize, max: usize }, + + /// Provider API returned an error. + #[error("transcription API error: {message}")] + ApiError { message: String }, + + /// Network or HTTP error. + #[error("transcription request failed: {0}")] + RequestFailed(String), + + /// Provider is not configured. + #[error("transcription provider not configured: {reason}")] + NotConfigured { reason: String }, +} + +/// Trait for speech-to-text transcription providers. +#[async_trait] +pub trait TranscriptionProvider: Send + Sync { + /// Provider name (e.g., "openai"). + fn name(&self) -> &str; + + /// Model name (e.g., "whisper-1"). + fn model_name(&self) -> &str; + + /// Maximum file size in bytes. + fn max_file_size(&self) -> usize; + + /// Supported audio formats. + fn supported_formats(&self) -> &[AudioFormat]; + + /// Transcribe audio bytes to text. + async fn transcribe( + &self, + audio: &[u8], + format: AudioFormat, + language: Option<&str>, + ) -> Result; +} + +/// Middleware that detects audio attachments and transcribes them. +pub struct TranscriptionMiddleware { + provider: Arc, + language: Option, +} + +impl TranscriptionMiddleware { + /// Create a new transcription middleware. + pub fn new(provider: Arc, language: Option) -> Self { + Self { provider, language } + } + + /// Process an incoming message, transcribing any audio attachments. + /// + /// If audio attachments are found and transcription succeeds, the message + /// content is replaced with the transcribed text. On failure, a fallback + /// message is used. + pub async fn process(&self, mut msg: IncomingMessage) -> IncomingMessage { + let audio_attachment = msg + .attachments + .iter() + .find(|a| a.kind == AttachmentKind::Audio); + + let Some(attachment) = audio_attachment else { + return msg; + }; + + let format = match AudioFormat::from_mime_type(&attachment.mime_type) { + Some(f) => f, + None => { + tracing::warn!( + mime = %attachment.mime_type, + "Unsupported audio format for transcription" + ); + if msg.content.is_empty() || msg.content == "[Voice note]" { + msg.content = "[Voice note: unsupported audio format]".to_string(); + } + return msg; + } + }; + + // Check size limit + if attachment.data.len() > self.provider.max_file_size() { + tracing::warn!( + size = attachment.data.len(), + max = self.provider.max_file_size(), + "Audio attachment exceeds provider size limit" + ); + if msg.content.is_empty() || msg.content == "[Voice note]" { + msg.content = "[Voice note: file too large for transcription]".to_string(); + } + return msg; + } + + match self + .provider + .transcribe(&attachment.data, format, self.language.as_deref()) + .await + { + Ok(text) => { + tracing::info!( + provider = %self.provider.name(), + text_len = text.len(), + "Audio transcription successful" + ); + msg.content = text; + } + Err(e) => { + tracing::error!( + error = %e, + provider = %self.provider.name(), + "Audio transcription failed" + ); + if msg.content.is_empty() || msg.content == "[Voice note]" { + msg.content = "[Voice note: transcription failed]".to_string(); + } + } + } + + msg + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::channels::Attachment; + + #[test] + fn audio_format_from_mime_type() { + assert_eq!( + AudioFormat::from_mime_type("audio/ogg"), + Some(AudioFormat::OggOpus) + ); + assert_eq!( + AudioFormat::from_mime_type("audio/ogg; codecs=opus"), + Some(AudioFormat::OggOpus) + ); + assert_eq!( + AudioFormat::from_mime_type("audio/mpeg"), + Some(AudioFormat::Mp3) + ); + assert_eq!( + AudioFormat::from_mime_type("audio/mp3"), + Some(AudioFormat::Mp3) + ); + assert_eq!( + AudioFormat::from_mime_type("audio/wav"), + Some(AudioFormat::Wav) + ); + assert_eq!( + AudioFormat::from_mime_type("audio/webm"), + Some(AudioFormat::Webm) + ); + assert_eq!( + AudioFormat::from_mime_type("audio/mp4"), + Some(AudioFormat::M4a) + ); + assert_eq!( + AudioFormat::from_mime_type("audio/m4a"), + Some(AudioFormat::M4a) + ); + assert_eq!(AudioFormat::from_mime_type("text/plain"), None); + assert_eq!(AudioFormat::from_mime_type(""), None); + } + + #[test] + fn audio_format_extension() { + assert_eq!(AudioFormat::OggOpus.extension(), "ogg"); + assert_eq!(AudioFormat::Mp3.extension(), "mp3"); + assert_eq!(AudioFormat::Wav.extension(), "wav"); + assert_eq!(AudioFormat::Webm.extension(), "webm"); + assert_eq!(AudioFormat::M4a.extension(), "m4a"); + } + + #[test] + fn audio_format_mime_type() { + assert_eq!(AudioFormat::OggOpus.mime_type(), "audio/ogg"); + assert_eq!(AudioFormat::Mp3.mime_type(), "audio/mpeg"); + } + + /// Mock provider for testing middleware. + struct MockProvider { + result: Result, + } + + #[async_trait] + impl TranscriptionProvider for MockProvider { + fn name(&self) -> &str { + "mock" + } + fn model_name(&self) -> &str { + "mock-1" + } + fn max_file_size(&self) -> usize { + 25 * 1024 * 1024 + } + fn supported_formats(&self) -> &[AudioFormat] { + &[AudioFormat::OggOpus, AudioFormat::Mp3] + } + async fn transcribe( + &self, + _audio: &[u8], + _format: AudioFormat, + _language: Option<&str>, + ) -> Result { + match &self.result { + Ok(text) => Ok(text.clone()), + Err(_) => Err(TranscriptionError::ApiError { + message: "mock error".to_string(), + }), + } + } + } + + #[tokio::test] + async fn middleware_transcribes_audio_attachment() { + let provider = Arc::new(MockProvider { + result: Ok("Hello, world!".to_string()), + }); + let middleware = TranscriptionMiddleware::new(provider, None); + + let msg = IncomingMessage::new("telegram", "user1", "[Voice note]").with_attachments(vec![ + Attachment { + kind: AttachmentKind::Audio, + mime_type: "audio/ogg".to_string(), + data: vec![0u8; 100], + filename: None, + duration_secs: Some(5), + }, + ]); + + let result = middleware.process(msg).await; + assert_eq!(result.content, "Hello, world!"); + } + + #[tokio::test] + async fn middleware_skips_non_audio_messages() { + let provider = Arc::new(MockProvider { + result: Ok("transcribed".to_string()), + }); + let middleware = TranscriptionMiddleware::new(provider, None); + + let msg = IncomingMessage::new("telegram", "user1", "regular text"); + let result = middleware.process(msg).await; + assert_eq!(result.content, "regular text"); + } + + #[tokio::test] + async fn middleware_handles_transcription_failure() { + let provider = Arc::new(MockProvider { + result: Err(TranscriptionError::ApiError { + message: "test".to_string(), + }), + }); + let middleware = TranscriptionMiddleware::new(provider, None); + + let msg = IncomingMessage::new("telegram", "user1", "[Voice note]").with_attachments(vec![ + Attachment { + kind: AttachmentKind::Audio, + mime_type: "audio/ogg".to_string(), + data: vec![0u8; 100], + filename: None, + duration_secs: Some(5), + }, + ]); + + let result = middleware.process(msg).await; + assert_eq!(result.content, "[Voice note: transcription failed]"); + } + + #[tokio::test] + async fn middleware_handles_unsupported_format() { + let provider = Arc::new(MockProvider { + result: Ok("text".to_string()), + }); + let middleware = TranscriptionMiddleware::new(provider, None); + + let msg = IncomingMessage::new("telegram", "user1", "[Voice note]").with_attachments(vec![ + Attachment { + kind: AttachmentKind::Audio, + mime_type: "audio/flac".to_string(), + data: vec![0u8; 100], + filename: None, + duration_secs: None, + }, + ]); + + let result = middleware.process(msg).await; + assert_eq!(result.content, "[Voice note: unsupported audio format]"); + } +} diff --git a/src/transcription/openai.rs b/src/transcription/openai.rs new file mode 100644 index 00000000..86902c5a --- /dev/null +++ b/src/transcription/openai.rs @@ -0,0 +1,196 @@ +//! OpenAI Whisper transcription provider. +//! +//! Uses the OpenAI `/v1/audio/transcriptions` endpoint with multipart form upload. + +use async_trait::async_trait; +use reqwest::multipart; +use secrecy::{ExposeSecret, SecretString}; + +use crate::transcription::{AudioFormat, TranscriptionError, TranscriptionProvider}; + +/// Maximum file size for Whisper API (25 MB). +const WHISPER_MAX_FILE_SIZE: usize = 25 * 1024 * 1024; + +/// Supported formats for Whisper. +const WHISPER_FORMATS: &[AudioFormat] = &[ + AudioFormat::OggOpus, + AudioFormat::Mp3, + AudioFormat::Wav, + AudioFormat::Webm, + AudioFormat::M4a, +]; + +/// OpenAI Whisper transcription provider. +pub struct OpenAiWhisper { + api_key: SecretString, + model: String, + base_url: String, + client: reqwest::Client, +} + +impl OpenAiWhisper { + /// Create a new OpenAI Whisper provider. + pub fn new(api_key: SecretString, model: String) -> Self { + Self { + api_key, + model, + base_url: "https://api.openai.com".to_string(), + client: reqwest::Client::new(), + } + } + + /// Set a custom base URL (for testing or alternative endpoints). + pub fn with_base_url(mut self, url: impl Into) -> Self { + self.base_url = url.into(); + self + } +} + +#[async_trait] +impl TranscriptionProvider for OpenAiWhisper { + fn name(&self) -> &str { + "openai" + } + + fn model_name(&self) -> &str { + &self.model + } + + fn max_file_size(&self) -> usize { + WHISPER_MAX_FILE_SIZE + } + + fn supported_formats(&self) -> &[AudioFormat] { + WHISPER_FORMATS + } + + async fn transcribe( + &self, + audio: &[u8], + format: AudioFormat, + language: Option<&str>, + ) -> Result { + if audio.len() > WHISPER_MAX_FILE_SIZE { + return Err(TranscriptionError::FileTooLarge { + size: audio.len(), + max: WHISPER_MAX_FILE_SIZE, + }); + } + + let filename = format!("audio.{}", format.extension()); + + // Note: to_vec() copies the audio bytes for the multipart body. + // Peak memory is ~2x the file size (original slice + copy). Acceptable + // for the 25 MB Whisper limit; revisit if supporting larger files. + let file_part = multipart::Part::bytes(audio.to_vec()) + .file_name(filename) + .mime_str(format.mime_type()) + .map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?; + + let mut form = multipart::Form::new() + .part("file", file_part) + .text("model", self.model.clone()) + .text("response_format", "text"); + + if let Some(lang) = language { + form = form.text("language", lang.to_string()); + } + + let url = format!("{}/v1/audio/transcriptions", self.base_url); + + let response = self + .client + .post(&url) + .header( + "Authorization", + format!("Bearer {}", self.api_key.expose_secret()), + ) + .multipart(form) + .send() + .await + .map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?; + + let status = response.status(); + let body = response + .text() + .await + .map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?; + + if !status.is_success() { + return Err(TranscriptionError::ApiError { + message: format!("HTTP {}: {}", status, body), + }); + } + + // response_format=text returns raw text, trim whitespace + Ok(body.trim().to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_too_large_is_rejected_sync() { + // The size check happens before any async work, so we can verify + // via a simple construction + manual call of the guard logic. + let oversized = vec![0u8; WHISPER_MAX_FILE_SIZE + 1]; + assert!(oversized.len() > WHISPER_MAX_FILE_SIZE); + } + + #[tokio::test] + async fn file_too_large_returns_error() { + let provider = OpenAiWhisper::new( + SecretString::from("sk-test".to_string()), + "whisper-1".to_string(), + ); + + let oversized = vec![0u8; WHISPER_MAX_FILE_SIZE + 1]; + let result = provider + .transcribe(&oversized, AudioFormat::Wav, None) + .await; + + match result { + Err(TranscriptionError::FileTooLarge { size, max }) => { + assert_eq!(size, WHISPER_MAX_FILE_SIZE + 1); + assert_eq!(max, WHISPER_MAX_FILE_SIZE); + } + other => panic!("Expected FileTooLarge, got: {:?}", other), + } + } + + #[tokio::test] + async fn api_error_on_bad_url() { + // Point at a URL that will fail to connect + let provider = OpenAiWhisper::new( + SecretString::from("sk-test".to_string()), + "whisper-1".to_string(), + ) + .with_base_url("http://127.0.0.1:1"); // port 1 won't be listening + + let audio = vec![0u8; 100]; + let result = provider + .transcribe(&audio, AudioFormat::OggOpus, Some("en")) + .await; + + assert!( + matches!(result, Err(TranscriptionError::RequestFailed(_))), + "Expected RequestFailed, got: {:?}", + result + ); + } + + #[test] + fn provider_metadata() { + let provider = OpenAiWhisper::new( + SecretString::from("sk-test".to_string()), + "whisper-1".to_string(), + ); + + assert_eq!(provider.name(), "openai"); + assert_eq!(provider.model_name(), "whisper-1"); + assert_eq!(provider.max_file_size(), WHISPER_MAX_FILE_SIZE); + assert_eq!(provider.supported_formats().len(), 5); + } +} diff --git a/wit/channel.wit b/wit/channel.wit index c716bc58..d9049030 100644 --- a/wit/channel.wit +++ b/wit/channel.wit @@ -113,6 +113,27 @@ interface channel-host { // ==================== Channel-Specific Capabilities ==================== + /// Kind of attachment (audio, image, document). + enum attachment-kind { + audio, + image, + document, + } + + /// Binary attachment on an emitted message (e.g., voice note, photo). + record attachment { + /// What kind of content this is. + kind: attachment-kind, + /// MIME type (e.g., "audio/ogg", "image/jpeg"). + mime-type: string, + /// Raw bytes of the attachment. + data: list, + /// Optional filename. + filename: option, + /// Duration in seconds (for audio/video). + duration-secs: option, + } + /// A message to emit to the agent. record emitted-message { /// User identifier within the channel (e.g., Slack user ID). @@ -125,6 +146,8 @@ interface channel-host { thread-id: option, /// Channel-specific metadata as JSON string. metadata-json: string, + /// Optional binary attachments (voice notes, images, etc.). + attachments: list, } /// Emit a message to the agent.