mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-25 14:53:34 +00:00
feat: add audio transcription pipeline with OpenAI Whisper and Telegram voice notes (#90)
Adds speech-to-text support so WASM channels can emit audio attachments that get automatically transcribed before reaching the agent. Telegram voice notes are the first integration — downloaded via Bot API and transcribed via OpenAI Whisper. - Extend WIT with attachment-kind, attachment records on emitted-message - Add Attachment/AttachmentKind types to channel.rs and IncomingMessage - Add TranscriptionProvider trait, AudioFormat enum, TranscriptionMiddleware - Implement OpenAI Whisper provider (multipart POST, 25MB limit) - Add TranscriptionConfig + TranscriptionSettings with env var overrides - Parse Telegram voice messages, download via getFile, emit as attachments - Apply transcription in both process/dispatch emitted message paths - Graceful degradation: download failures show "[Voice note: download failed]" - Validate attachment sizes (10MB max), drop oversized without losing message Co-Authored-By: Claude Opus 4.6 <[email protected]>
This commit is contained in:
serrrfirat
co-authored by
Claude Opus 4.6
Claude Opus 4.6
parent
b68d67bd35
commit
bbb2d5c4dd
+1
-1
@@ -41,7 +41,7 @@ tokio-stream = { version = "0.1", features = ["sync"] }
|
||||
futures = "0.3"
|
||||
|
||||
# HTTP client
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls-native-roots", "stream"] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "multipart", "rustls-tls-native-roots", "stream"] }
|
||||
|
||||
# Serialization
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
|
||||
@@ -25,5 +25,3 @@ opt-level = "s"
|
||||
lto = true
|
||||
strip = true
|
||||
codegen-units = 1
|
||||
|
||||
[workspace]
|
||||
|
||||
@@ -76,6 +76,9 @@ struct TelegramMessage {
|
||||
#[serde(default)]
|
||||
caption: Option<String>,
|
||||
|
||||
/// Voice message.
|
||||
voice: Option<TelegramVoice>,
|
||||
|
||||
/// Original message if this is a reply.
|
||||
reply_to_message: Option<Box<TelegramMessage>>,
|
||||
|
||||
@@ -139,6 +142,36 @@ struct MessageEntity {
|
||||
user: Option<TelegramUser>,
|
||||
}
|
||||
|
||||
/// Telegram Voice object.
|
||||
/// https://core.telegram.org/bots/api#voice
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TelegramVoice {
|
||||
/// Identifier for this file, which can be used to download the file.
|
||||
file_id: String,
|
||||
|
||||
/// Duration of the audio in seconds.
|
||||
duration: u32,
|
||||
|
||||
/// MIME type of the file.
|
||||
#[serde(default)]
|
||||
mime_type: Option<String>,
|
||||
|
||||
/// File size in bytes.
|
||||
#[serde(default)]
|
||||
file_size: Option<i64>,
|
||||
}
|
||||
|
||||
/// Telegram File object returned by getFile.
|
||||
/// https://core.telegram.org/bots/api#file
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TelegramFile {
|
||||
/// Identifier for this file.
|
||||
file_id: String,
|
||||
|
||||
/// File path for downloading. Use https://api.telegram.org/file/bot<token>/<file_path>.
|
||||
file_path: Option<String>,
|
||||
}
|
||||
|
||||
/// Telegram API response wrapper.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TelegramApiResponse<T> {
|
||||
@@ -867,6 +900,75 @@ fn send_pairing_reply(chat_id: i64, code: &str) -> Result<(), String> {
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Voice File Download
|
||||
// ============================================================================
|
||||
|
||||
/// Download a voice file from Telegram by file_id.
|
||||
///
|
||||
/// 1. Call getFile to get the file_path.
|
||||
/// 2. Download the file bytes from /file/bot{TOKEN}/{file_path}.
|
||||
fn download_voice_file(file_id: &str) -> Result<Vec<u8>, String> {
|
||||
// Step 1: Call getFile to get file_path
|
||||
// Double braces `{{...}}` produce a literal `{TELEGRAM_BOT_TOKEN}` placeholder
|
||||
// in the URL, which the host-side credential injector replaces with the real token.
|
||||
let get_file_url = format!(
|
||||
"https://api.telegram.org/bot{{TELEGRAM_BOT_TOKEN}}/getFile?file_id={}",
|
||||
file_id
|
||||
);
|
||||
|
||||
let headers = serde_json::json!({});
|
||||
let result = channel_host::http_request("GET", &get_file_url, &headers.to_string(), None, None);
|
||||
|
||||
let response = result.map_err(|e| format!("getFile request failed: {}", e))?;
|
||||
|
||||
if response.status != 200 {
|
||||
let body_str = String::from_utf8_lossy(&response.body);
|
||||
return Err(format!("getFile returned {}: {}", response.status, body_str));
|
||||
}
|
||||
|
||||
let api_response: TelegramApiResponse<TelegramFile> =
|
||||
serde_json::from_slice(&response.body)
|
||||
.map_err(|e| format!("Failed to parse getFile response: {}", e))?;
|
||||
|
||||
if !api_response.ok {
|
||||
return Err(format!(
|
||||
"getFile API error: {}",
|
||||
api_response
|
||||
.description
|
||||
.unwrap_or_else(|| "unknown".to_string())
|
||||
));
|
||||
}
|
||||
|
||||
let file = api_response
|
||||
.result
|
||||
.ok_or_else(|| "getFile returned no result".to_string())?;
|
||||
|
||||
let file_path = file
|
||||
.file_path
|
||||
.ok_or_else(|| "getFile returned no file_path".to_string())?;
|
||||
|
||||
// Step 2: Download the actual file bytes
|
||||
let download_url = format!(
|
||||
"https://api.telegram.org/file/bot{{TELEGRAM_BOT_TOKEN}}/{}",
|
||||
file_path
|
||||
);
|
||||
|
||||
let result =
|
||||
channel_host::http_request("GET", &download_url, &headers.to_string(), None, None);
|
||||
|
||||
let response = result.map_err(|e| format!("File download failed: {}", e))?;
|
||||
|
||||
if response.status != 200 {
|
||||
return Err(format!(
|
||||
"File download returned status {}",
|
||||
response.status
|
||||
));
|
||||
}
|
||||
|
||||
Ok(response.body)
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Update Handling
|
||||
// ============================================================================
|
||||
@@ -886,6 +988,9 @@ fn handle_update(update: TelegramUpdate) {
|
||||
|
||||
/// Process a single message.
|
||||
fn handle_message(message: TelegramMessage) {
|
||||
// Check for voice note first (voice-only messages have no text)
|
||||
let is_voice = message.voice.is_some();
|
||||
|
||||
// Use text or caption (for media messages)
|
||||
let content = message
|
||||
.text
|
||||
@@ -893,7 +998,8 @@ fn handle_message(message: TelegramMessage) {
|
||||
.or_else(|| message.caption.filter(|c| !c.is_empty()))
|
||||
.unwrap_or_default();
|
||||
|
||||
if content.is_empty() {
|
||||
// Allow voice notes through even when content is empty
|
||||
if content.is_empty() && !is_voice {
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -1038,19 +1144,65 @@ fn handle_message(message: TelegramMessage) {
|
||||
},
|
||||
);
|
||||
|
||||
// Handle voice notes: download and attach audio bytes.
|
||||
// Note: download is synchronous (two HTTP roundtrips to Telegram API).
|
||||
// This blocks the WASM execution for the current polling tick.
|
||||
let mut attachments = Vec::new();
|
||||
let mut voice_download_failed = false;
|
||||
if let Some(ref voice) = message.voice {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!(
|
||||
"Voice note from user {} (duration: {}s, file_id: {})",
|
||||
from.id, voice.duration, voice.file_id
|
||||
),
|
||||
);
|
||||
|
||||
match download_voice_file(&voice.file_id) {
|
||||
Ok(audio_bytes) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Info,
|
||||
&format!("Downloaded voice file: {} bytes", audio_bytes.len()),
|
||||
);
|
||||
attachments.push(channel_host::Attachment {
|
||||
kind: channel_host::AttachmentKind::Audio,
|
||||
mime_type: voice
|
||||
.mime_type
|
||||
.clone()
|
||||
.unwrap_or_else(|| "audio/ogg".to_string()),
|
||||
data: audio_bytes,
|
||||
filename: Some(format!("voice_{}.ogg", voice.file_id)),
|
||||
duration_secs: Some(voice.duration),
|
||||
});
|
||||
}
|
||||
Err(e) => {
|
||||
channel_host::log(
|
||||
channel_host::LogLevel::Error,
|
||||
&format!("Failed to download voice file: {}", e),
|
||||
);
|
||||
voice_download_failed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Determine what to emit to the agent.
|
||||
// - Voice notes: use "[Voice note]" as content (transcription happens host-side)
|
||||
// - `/start` (no args): emit a welcome placeholder so the agent greets the user
|
||||
// - Other bare `/commands` (e.g. /interrupt, /help): pass the raw command through
|
||||
// so Submission::parse() can handle it
|
||||
// - Commands with args (e.g. `/start hello`): cleaned_text already has the args
|
||||
// - Plain text: pass through as-is
|
||||
let trimmed_content = content.trim();
|
||||
let content_to_emit = if trimmed_content.eq_ignore_ascii_case("/start") {
|
||||
let content_to_emit = if is_voice && voice_download_failed && content.is_empty() {
|
||||
"[Voice note: download failed]".to_string()
|
||||
} else if is_voice && content.is_empty() {
|
||||
"[Voice note]".to_string()
|
||||
} else if trimmed_content.eq_ignore_ascii_case("/start") {
|
||||
"[User started the bot]".to_string()
|
||||
} else if cleaned_text.is_empty() && trimmed_content.starts_with('/') {
|
||||
// Bare control command like /interrupt, /stop, /help — pass through raw
|
||||
trimmed_content.to_string()
|
||||
} else if cleaned_text.is_empty() {
|
||||
} else if cleaned_text.is_empty() && !is_voice {
|
||||
return;
|
||||
} else {
|
||||
cleaned_text
|
||||
@@ -1063,6 +1215,7 @@ fn handle_message(message: TelegramMessage) {
|
||||
content: content_to_emit,
|
||||
thread_id: None, // Telegram doesn't have threads in the same way
|
||||
metadata_json,
|
||||
attachments,
|
||||
});
|
||||
|
||||
channel_host::log(
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"type":"channel","name":"telegram","description":"Telegram Bot API channel for receiving and responding to Telegram messages","capabilities":{"http":{"allowlist":[{"host":"api.telegram.org","path_prefix":"/bot"}],"credentials":{"telegram_bot":{"secret_name":"telegram_bot_token","location":{"type":"url_path","placeholder":"{TELEGRAM_BOT_TOKEN}"},"host_patterns":["api.telegram.org"]}},"rate_limit":{"requests_per_minute":30,"requests_per_hour":1000}},"secrets":{"allowed_names":["telegram_*"]},"channel":{"allowed_paths":["/webhook/telegram"],"allow_polling":true,"min_poll_interval_ms":30000,"workspace_prefix":"channels/telegram/","emit_rate_limit":{"messages_per_minute":100,"messages_per_hour":5000}}},"config":{"bot_username":null,"owner_id":null,"respond_to_all_group_messages":false,"polling_enabled":false,"poll_interval_ms":30000,"dm_policy":"pairing","allow_from":[]}}
|
||||
{"type":"channel","name":"telegram","description":"Telegram Bot API channel for receiving and responding to Telegram messages","capabilities":{"http":{"allowlist":[{"host":"api.telegram.org","path_prefix":"/bot"},{"host":"api.telegram.org","path_prefix":"/file/bot"}],"credentials":{"telegram_bot":{"secret_name":"telegram_bot_token","location":{"type":"url_path","placeholder":"{TELEGRAM_BOT_TOKEN}"},"host_patterns":["api.telegram.org"]}},"rate_limit":{"requests_per_minute":30,"requests_per_hour":1000}},"secrets":{"allowed_names":["telegram_*"]},"channel":{"allowed_paths":["/webhook/telegram"],"allow_polling":true,"min_poll_interval_ms":30000,"workspace_prefix":"channels/telegram/","emit_rate_limit":{"messages_per_minute":100,"messages_per_hour":5000}}},"config":{"bot_username":null,"owner_id":null,"respond_to_all_group_messages":false,"polling_enabled":false,"poll_interval_ms":30000,"dm_policy":"pairing","allow_from":[]}}
|
||||
|
||||
@@ -9,6 +9,32 @@ use uuid::Uuid;
|
||||
|
||||
use crate::error::ChannelError;
|
||||
|
||||
/// Kind of attachment carried on an incoming message.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum AttachmentKind {
|
||||
/// Audio content (voice notes, audio files).
|
||||
Audio,
|
||||
/// Image content (photos, screenshots).
|
||||
Image,
|
||||
/// Document content (PDFs, files).
|
||||
Document,
|
||||
}
|
||||
|
||||
/// Binary attachment on a message (e.g., voice note, photo).
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Attachment {
|
||||
/// What kind of content this is.
|
||||
pub kind: AttachmentKind,
|
||||
/// MIME type (e.g., "audio/ogg", "image/jpeg").
|
||||
pub mime_type: String,
|
||||
/// Raw bytes of the attachment.
|
||||
pub data: Vec<u8>,
|
||||
/// Optional filename.
|
||||
pub filename: Option<String>,
|
||||
/// Duration in seconds (for audio/video).
|
||||
pub duration_secs: Option<u32>,
|
||||
}
|
||||
|
||||
/// A message received from an external channel.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct IncomingMessage {
|
||||
@@ -28,6 +54,8 @@ pub struct IncomingMessage {
|
||||
pub received_at: DateTime<Utc>,
|
||||
/// Channel-specific metadata.
|
||||
pub metadata: serde_json::Value,
|
||||
/// Binary attachments (voice notes, images, etc.).
|
||||
pub attachments: Vec<Attachment>,
|
||||
}
|
||||
|
||||
impl IncomingMessage {
|
||||
@@ -46,6 +74,7 @@ impl IncomingMessage {
|
||||
thread_id: None,
|
||||
received_at: Utc::now(),
|
||||
metadata: serde_json::Value::Null,
|
||||
attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,6 +95,12 @@ impl IncomingMessage {
|
||||
self.user_name = Some(name.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Set attachments.
|
||||
pub fn with_attachments(mut self, attachments: Vec<Attachment>) -> Self {
|
||||
self.attachments = attachments;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream of incoming messages.
|
||||
|
||||
+4
-1
@@ -35,7 +35,10 @@ pub mod wasm;
|
||||
pub mod web;
|
||||
mod webhook_server;
|
||||
|
||||
pub use channel::{Channel, IncomingMessage, MessageStream, OutgoingResponse, StatusUpdate};
|
||||
pub use channel::{
|
||||
Attachment, AttachmentKind, Channel, IncomingMessage, MessageStream, OutgoingResponse,
|
||||
StatusUpdate,
|
||||
};
|
||||
pub use http::HttpChannel;
|
||||
pub use manager::ChannelManager;
|
||||
pub use repl::ReplChannel;
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use crate::channels::channel::Attachment;
|
||||
use crate::channels::wasm::capabilities::{ChannelCapabilities, EmitRateLimitConfig};
|
||||
use crate::channels::wasm::error::WasmChannelError;
|
||||
use crate::tools::wasm::{HostState, LogLevel};
|
||||
@@ -17,6 +18,9 @@ const MAX_EMITS_PER_EXECUTION: usize = 100;
|
||||
/// Maximum message content size (64 KB).
|
||||
const MAX_MESSAGE_CONTENT_SIZE: usize = 64 * 1024;
|
||||
|
||||
/// Maximum size for a single attachment (10 MB).
|
||||
const MAX_ATTACHMENT_SIZE: usize = 10 * 1024 * 1024;
|
||||
|
||||
/// A message emitted by a WASM channel to be sent to the agent.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EmittedMessage {
|
||||
@@ -37,6 +41,9 @@ pub struct EmittedMessage {
|
||||
|
||||
/// Timestamp when the message was emitted.
|
||||
pub emitted_at_millis: u64,
|
||||
|
||||
/// Binary attachments (voice notes, images, etc.).
|
||||
pub attachments: Vec<Attachment>,
|
||||
}
|
||||
|
||||
impl EmittedMessage {
|
||||
@@ -52,6 +59,7 @@ impl EmittedMessage {
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|d| d.as_millis() as u64)
|
||||
.unwrap_or(0),
|
||||
attachments: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,6 +80,12 @@ impl EmittedMessage {
|
||||
self.metadata_json = metadata_json.into();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set attachments.
|
||||
pub fn with_attachments(mut self, attachments: Vec<Attachment>) -> Self {
|
||||
self.attachments = attachments;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// A pending workspace write operation.
|
||||
@@ -168,7 +182,7 @@ impl ChannelHostState {
|
||||
///
|
||||
/// Messages are queued and delivered after callback execution completes.
|
||||
/// Rate limiting is enforced per-execution and globally.
|
||||
pub fn emit_message(&mut self, msg: EmittedMessage) -> Result<(), WasmChannelError> {
|
||||
pub fn emit_message(&mut self, mut msg: EmittedMessage) -> Result<(), WasmChannelError> {
|
||||
// Check per-execution limit
|
||||
if !self.emit_enabled {
|
||||
self.emits_dropped += 1;
|
||||
@@ -186,6 +200,22 @@ impl ChannelHostState {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Validate attachment sizes — drop only oversized attachments, not the whole message
|
||||
msg.attachments.retain(|attachment| {
|
||||
if attachment.data.len() > MAX_ATTACHMENT_SIZE {
|
||||
tracing::warn!(
|
||||
channel = %self.channel_name,
|
||||
size = attachment.data.len(),
|
||||
max = MAX_ATTACHMENT_SIZE,
|
||||
mime = %attachment.mime_type,
|
||||
"Attachment too large, dropping attachment (message still delivered)"
|
||||
);
|
||||
false
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
|
||||
// Validate message content size
|
||||
if msg.content.len() > MAX_MESSAGE_CONTENT_SIZE {
|
||||
tracing::warn!(
|
||||
|
||||
@@ -40,6 +40,7 @@ use wasmtime::Store;
|
||||
use wasmtime::component::Linker;
|
||||
use wasmtime_wasi::{ResourceTable, WasiCtx, WasiCtxBuilder, WasiView};
|
||||
|
||||
use crate::channels::channel::{Attachment, AttachmentKind};
|
||||
use crate::channels::wasm::capabilities::ChannelCapabilities;
|
||||
use crate::channels::wasm::error::WasmChannelError;
|
||||
use crate::channels::wasm::host::{
|
||||
@@ -436,6 +437,7 @@ impl near::agent::channel_host::Host for ChannelStoreData {
|
||||
user_id = %msg.user_id,
|
||||
user_name = ?msg.user_name,
|
||||
content_len = msg.content.len(),
|
||||
attachment_count = msg.attachments.len(),
|
||||
"WASM emit_message called"
|
||||
);
|
||||
|
||||
@@ -448,6 +450,31 @@ impl near::agent::channel_host::Host for ChannelStoreData {
|
||||
}
|
||||
emitted = emitted.with_metadata(msg.metadata_json);
|
||||
|
||||
// Convert WIT attachments to Rust types
|
||||
if !msg.attachments.is_empty() {
|
||||
let attachments = msg
|
||||
.attachments
|
||||
.into_iter()
|
||||
.map(|a| {
|
||||
let kind = match a.kind {
|
||||
near::agent::channel_host::AttachmentKind::Audio => AttachmentKind::Audio,
|
||||
near::agent::channel_host::AttachmentKind::Image => AttachmentKind::Image,
|
||||
near::agent::channel_host::AttachmentKind::Document => {
|
||||
AttachmentKind::Document
|
||||
}
|
||||
};
|
||||
Attachment {
|
||||
kind,
|
||||
mime_type: a.mime_type,
|
||||
data: a.data,
|
||||
filename: a.filename,
|
||||
duration_secs: a.duration_secs,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
emitted = emitted.with_attachments(attachments);
|
||||
}
|
||||
|
||||
match self.host_state.emit_message(emitted) {
|
||||
Ok(()) => {
|
||||
tracing::info!("Message emitted to host state successfully");
|
||||
@@ -553,6 +580,9 @@ pub struct WasmChannel {
|
||||
/// In-memory workspace store persisting writes across callback invocations.
|
||||
/// Ensures WASM channels can maintain state (e.g., polling offsets) between ticks.
|
||||
workspace_store: Arc<ChannelWorkspaceStore>,
|
||||
|
||||
/// Optional transcription middleware for audio attachments.
|
||||
transcription_middleware: Option<Arc<crate::transcription::TranscriptionMiddleware>>,
|
||||
}
|
||||
|
||||
impl WasmChannel {
|
||||
@@ -584,9 +614,18 @@ impl WasmChannel {
|
||||
typing_task: RwLock::new(None),
|
||||
pairing_store,
|
||||
workspace_store: Arc::new(ChannelWorkspaceStore::new()),
|
||||
transcription_middleware: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the transcription middleware for audio attachment processing.
|
||||
pub fn set_transcription_middleware(
|
||||
&mut self,
|
||||
middleware: Arc<crate::transcription::TranscriptionMiddleware>,
|
||||
) {
|
||||
self.transcription_middleware = Some(middleware);
|
||||
}
|
||||
|
||||
/// Update the channel config before starting.
|
||||
///
|
||||
/// Merges the provided values into the existing config JSON.
|
||||
@@ -1510,11 +1549,21 @@ impl WasmChannel {
|
||||
msg = msg.with_metadata(metadata);
|
||||
}
|
||||
|
||||
// Send to stream
|
||||
// Carry attachments through (moves emitted.attachments into msg)
|
||||
if !emitted.attachments.is_empty() {
|
||||
msg = msg.with_attachments(emitted.attachments);
|
||||
}
|
||||
|
||||
// Apply transcription middleware if available (may replace content with transcript)
|
||||
if let Some(ref middleware) = self.transcription_middleware {
|
||||
msg = middleware.process(msg).await;
|
||||
}
|
||||
|
||||
// Send to stream (log post-transcription state intentionally)
|
||||
tracing::info!(
|
||||
channel = %self.name,
|
||||
user_id = %emitted.user_id,
|
||||
content_len = emitted.content.len(),
|
||||
user_id = %msg.user_id,
|
||||
content_len = msg.content.len(),
|
||||
"Sending emitted message to agent"
|
||||
);
|
||||
|
||||
@@ -1551,6 +1600,7 @@ impl WasmChannel {
|
||||
let pairing_store = self.pairing_store.clone();
|
||||
let callback_timeout = self.runtime.config().callback_timeout;
|
||||
let workspace_store = self.workspace_store.clone();
|
||||
let transcription_middleware = self.transcription_middleware.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval_timer = tokio::time::interval(interval);
|
||||
@@ -1585,6 +1635,7 @@ impl WasmChannel {
|
||||
emitted_messages,
|
||||
&message_tx,
|
||||
&rate_limiter,
|
||||
transcription_middleware.as_deref(),
|
||||
).await {
|
||||
tracing::warn!(
|
||||
channel = %channel_name,
|
||||
@@ -1708,6 +1759,7 @@ impl WasmChannel {
|
||||
messages: Vec<EmittedMessage>,
|
||||
message_tx: &RwLock<Option<mpsc::Sender<IncomingMessage>>>,
|
||||
rate_limiter: &RwLock<ChannelEmitRateLimiter>,
|
||||
transcription_middleware: Option<&crate::transcription::TranscriptionMiddleware>,
|
||||
) -> Result<(), WasmChannelError> {
|
||||
tracing::info!(
|
||||
channel = %channel_name,
|
||||
@@ -1755,11 +1807,21 @@ impl WasmChannel {
|
||||
msg = msg.with_metadata(metadata);
|
||||
}
|
||||
|
||||
// Send to stream
|
||||
// Carry attachments through (moves emitted.attachments into msg)
|
||||
if !emitted.attachments.is_empty() {
|
||||
msg = msg.with_attachments(emitted.attachments);
|
||||
}
|
||||
|
||||
// Apply transcription middleware if available (may replace content with transcript)
|
||||
if let Some(middleware) = transcription_middleware {
|
||||
msg = middleware.process(msg).await;
|
||||
}
|
||||
|
||||
// Send to stream (log post-transcription state intentionally)
|
||||
tracing::info!(
|
||||
channel = %channel_name,
|
||||
user_id = %emitted.user_id,
|
||||
content_len = emitted.content.len(),
|
||||
user_id = %msg.user_id,
|
||||
content_len = msg.content.len(),
|
||||
"Sending polled message to agent"
|
||||
);
|
||||
|
||||
@@ -2332,6 +2394,7 @@ mod tests {
|
||||
messages,
|
||||
&message_tx,
|
||||
&rate_limiter,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -2370,6 +2433,7 @@ mod tests {
|
||||
messages,
|
||||
&message_tx,
|
||||
&rate_limiter,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@ mod safety;
|
||||
mod sandbox;
|
||||
mod secrets;
|
||||
mod skills;
|
||||
mod transcription;
|
||||
mod tunnel;
|
||||
mod wasm;
|
||||
|
||||
@@ -45,6 +46,7 @@ pub use self::safety::SafetyConfig;
|
||||
pub use self::sandbox::{ClaudeCodeConfig, SandboxModeConfig};
|
||||
pub use self::secrets::SecretsConfig;
|
||||
pub use self::skills::SkillsConfig;
|
||||
pub use self::transcription::TranscriptionConfig;
|
||||
pub use self::tunnel::TunnelConfig;
|
||||
pub use self::wasm::WasmConfig;
|
||||
|
||||
@@ -74,6 +76,7 @@ pub struct Config {
|
||||
pub sandbox: SandboxModeConfig,
|
||||
pub claude_code: ClaudeCodeConfig,
|
||||
pub skills: SkillsConfig,
|
||||
pub transcription: TranscriptionConfig,
|
||||
pub observability: crate::observability::ObservabilityConfig,
|
||||
}
|
||||
|
||||
@@ -198,6 +201,7 @@ impl Config {
|
||||
sandbox: SandboxModeConfig::resolve()?,
|
||||
claude_code: ClaudeCodeConfig::resolve()?,
|
||||
skills: SkillsConfig::resolve()?,
|
||||
transcription: TranscriptionConfig::resolve(settings)?,
|
||||
observability: crate::observability::ObservabilityConfig {
|
||||
backend: std::env::var("OBSERVABILITY_BACKEND").unwrap_or_else(|_| "none".into()),
|
||||
},
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
use secrecy::SecretString;
|
||||
|
||||
use crate::config::helpers::optional_env;
|
||||
use crate::error::ConfigError;
|
||||
use crate::settings::Settings;
|
||||
|
||||
/// Transcription provider configuration.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TranscriptionConfig {
|
||||
/// Whether transcription is enabled.
|
||||
pub enabled: bool,
|
||||
/// Provider to use: "openai".
|
||||
pub provider: String,
|
||||
/// OpenAI API key (reused from embeddings/LLM config).
|
||||
pub openai_api_key: Option<SecretString>,
|
||||
/// Model to use for transcription.
|
||||
pub model: String,
|
||||
/// Optional language hint (ISO-639-1, e.g., "en").
|
||||
pub language: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for TranscriptionConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
provider: "openai".to_string(),
|
||||
openai_api_key: None,
|
||||
model: "whisper-1".to_string(),
|
||||
language: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl TranscriptionConfig {
|
||||
pub(crate) fn resolve(settings: &Settings) -> Result<Self, ConfigError> {
|
||||
let openai_api_key = optional_env("OPENAI_API_KEY")?.map(SecretString::from);
|
||||
|
||||
let provider = optional_env("TRANSCRIPTION_PROVIDER")?
|
||||
.unwrap_or_else(|| settings.transcription.provider.clone());
|
||||
|
||||
let model = optional_env("TRANSCRIPTION_MODEL")?
|
||||
.unwrap_or_else(|| settings.transcription.model.clone());
|
||||
|
||||
let language = optional_env("TRANSCRIPTION_LANGUAGE")?
|
||||
.or_else(|| settings.transcription.language.clone());
|
||||
|
||||
let enabled = optional_env("TRANSCRIPTION_ENABLED")?
|
||||
.map(|s| s.parse())
|
||||
.transpose()
|
||||
.map_err(|e| ConfigError::InvalidValue {
|
||||
key: "TRANSCRIPTION_ENABLED".to_string(),
|
||||
message: format!("must be 'true' or 'false': {e}"),
|
||||
})?
|
||||
.unwrap_or(settings.transcription.enabled);
|
||||
|
||||
// Only "openai" is currently supported
|
||||
if enabled && provider != "openai" {
|
||||
return Err(ConfigError::InvalidValue {
|
||||
key: "TRANSCRIPTION_PROVIDER".to_string(),
|
||||
message: format!(
|
||||
"unsupported provider '{}', only 'openai' is currently supported",
|
||||
provider
|
||||
),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(Self {
|
||||
enabled,
|
||||
provider,
|
||||
openai_api_key,
|
||||
model,
|
||||
language,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::config::helpers::ENV_MUTEX;
|
||||
use crate::settings::{Settings, TranscriptionSettings};
|
||||
|
||||
fn clear_transcription_env() {
|
||||
unsafe {
|
||||
std::env::remove_var("TRANSCRIPTION_ENABLED");
|
||||
std::env::remove_var("TRANSCRIPTION_PROVIDER");
|
||||
std::env::remove_var("TRANSCRIPTION_MODEL");
|
||||
std::env::remove_var("TRANSCRIPTION_LANGUAGE");
|
||||
std::env::remove_var("OPENAI_API_KEY");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_defaults_from_settings() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
let settings = Settings::default();
|
||||
let config = TranscriptionConfig::resolve(&settings).expect("resolve should succeed");
|
||||
|
||||
assert!(!config.enabled);
|
||||
assert_eq!(config.provider, "openai");
|
||||
assert_eq!(config.model, "whisper-1");
|
||||
assert!(config.language.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_env_overrides_settings() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
unsafe {
|
||||
std::env::set_var("TRANSCRIPTION_ENABLED", "true");
|
||||
std::env::set_var("TRANSCRIPTION_MODEL", "whisper-large-v3");
|
||||
std::env::set_var("TRANSCRIPTION_LANGUAGE", "en");
|
||||
}
|
||||
|
||||
let settings = Settings::default();
|
||||
let config = TranscriptionConfig::resolve(&settings).expect("resolve should succeed");
|
||||
|
||||
assert!(config.enabled);
|
||||
assert_eq!(config.model, "whisper-large-v3");
|
||||
assert_eq!(config.language, Some("en".to_string()));
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var("TRANSCRIPTION_ENABLED");
|
||||
std::env::remove_var("TRANSCRIPTION_MODEL");
|
||||
std::env::remove_var("TRANSCRIPTION_LANGUAGE");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_settings_with_custom_values() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
let settings = Settings {
|
||||
transcription: TranscriptionSettings {
|
||||
enabled: true,
|
||||
model: "whisper-large-v3".to_string(),
|
||||
language: Some("fr".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let config = TranscriptionConfig::resolve(&settings).expect("resolve should succeed");
|
||||
|
||||
assert!(config.enabled);
|
||||
assert_eq!(config.model, "whisper-large-v3");
|
||||
assert_eq!(config.language, Some("fr".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_rejects_unsupported_provider() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
let settings = Settings {
|
||||
transcription: TranscriptionSettings {
|
||||
enabled: true,
|
||||
provider: "deepgram".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let result = TranscriptionConfig::resolve(&settings);
|
||||
assert!(result.is_err());
|
||||
let err_msg = result.unwrap_err().to_string();
|
||||
assert!(
|
||||
err_msg.contains("unsupported provider"),
|
||||
"Error should mention unsupported provider, got: {err_msg}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transcription_disabled_skips_provider_validation() {
|
||||
let _guard = ENV_MUTEX.lock().expect("env mutex poisoned");
|
||||
clear_transcription_env();
|
||||
|
||||
// When disabled, any provider string is accepted (never used)
|
||||
let settings = Settings {
|
||||
transcription: TranscriptionSettings {
|
||||
enabled: false,
|
||||
provider: "nonexistent".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let config = TranscriptionConfig::resolve(&settings).expect("should succeed when disabled");
|
||||
assert!(!config.enabled);
|
||||
}
|
||||
}
|
||||
@@ -67,6 +67,7 @@ pub mod setup;
|
||||
pub mod skills;
|
||||
pub mod tools;
|
||||
pub mod tracing_fmt;
|
||||
pub mod transcription;
|
||||
pub mod tunnel;
|
||||
pub mod util;
|
||||
pub mod worker;
|
||||
|
||||
+31
-1
@@ -1028,6 +1028,32 @@ async fn main() -> anyhow::Result<()> {
|
||||
// Collect webhook route fragments; a single WebhookServer hosts them all.
|
||||
let mut webhook_routes: Vec<axum::Router> = Vec::new();
|
||||
|
||||
// Build transcription middleware if enabled.
|
||||
let transcription_middleware: Option<Arc<ironclaw::transcription::TranscriptionMiddleware>> =
|
||||
if config.transcription.enabled {
|
||||
if let Some(ref api_key) = config.transcription.openai_api_key {
|
||||
let provider = Arc::new(ironclaw::transcription::openai::OpenAiWhisper::new(
|
||||
api_key.clone(),
|
||||
config.transcription.model.clone(),
|
||||
));
|
||||
let middleware = ironclaw::transcription::TranscriptionMiddleware::new(
|
||||
provider,
|
||||
config.transcription.language.clone(),
|
||||
);
|
||||
tracing::info!(
|
||||
provider = %config.transcription.provider,
|
||||
model = %config.transcription.model,
|
||||
"Audio transcription enabled"
|
||||
);
|
||||
Some(Arc::new(middleware))
|
||||
} else {
|
||||
tracing::warn!("Transcription enabled but OPENAI_API_KEY not set, disabling");
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Load WASM channels and register their webhook routes.
|
||||
if config.channels.wasm_channels_enabled && config.channels.wasm_channels_dir.exists() {
|
||||
match WasmChannelRuntime::new(WasmChannelRuntimeConfig::default()) {
|
||||
@@ -1072,7 +1098,11 @@ async fn main() -> anyhow::Result<()> {
|
||||
require_secret: webhook_secret.is_some(),
|
||||
}];
|
||||
|
||||
let channel_arc = Arc::new(loaded.channel);
|
||||
let mut channel = loaded.channel;
|
||||
if let Some(ref mw) = transcription_middleware {
|
||||
channel.set_transcription_middleware(Arc::clone(mw));
|
||||
}
|
||||
let channel_arc = Arc::new(channel);
|
||||
|
||||
{
|
||||
let mut config_updates = std::collections::HashMap::new();
|
||||
|
||||
@@ -63,6 +63,11 @@ pub struct Settings {
|
||||
#[serde(default)]
|
||||
pub embeddings: EmbeddingsSettings,
|
||||
|
||||
// === Transcription (STT) ===
|
||||
/// Transcription configuration for voice notes.
|
||||
#[serde(default)]
|
||||
pub transcription: TranscriptionSettings,
|
||||
|
||||
// === Step 6: Channels ===
|
||||
/// Tunnel configuration for public webhook endpoints.
|
||||
#[serde(default)]
|
||||
@@ -146,6 +151,45 @@ impl Default for EmbeddingsSettings {
|
||||
}
|
||||
}
|
||||
|
||||
/// Transcription (STT) configuration for voice notes.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TranscriptionSettings {
|
||||
/// Whether transcription is enabled.
|
||||
#[serde(default)]
|
||||
pub enabled: bool,
|
||||
|
||||
/// Provider to use: "openai".
|
||||
#[serde(default = "default_transcription_provider")]
|
||||
pub provider: String,
|
||||
|
||||
/// Model to use for transcription.
|
||||
#[serde(default = "default_transcription_model")]
|
||||
pub model: String,
|
||||
|
||||
/// Optional language hint (ISO-639-1, e.g., "en").
|
||||
#[serde(default)]
|
||||
pub language: Option<String>,
|
||||
}
|
||||
|
||||
fn default_transcription_provider() -> String {
|
||||
"openai".to_string()
|
||||
}
|
||||
|
||||
fn default_transcription_model() -> String {
|
||||
"whisper-1".to_string()
|
||||
}
|
||||
|
||||
impl Default for TranscriptionSettings {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: false,
|
||||
provider: default_transcription_provider(),
|
||||
model: default_transcription_model(),
|
||||
language: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tunnel settings for public webhook endpoints.
|
||||
///
|
||||
/// The tunnel URL is shared across all channels that need webhooks.
|
||||
|
||||
@@ -0,0 +1,375 @@
|
||||
//! Audio transcription for voice notes and audio attachments.
|
||||
//!
|
||||
//! Provides a `TranscriptionProvider` trait and middleware that automatically
|
||||
//! transcribes audio attachments on incoming messages before they reach the agent.
|
||||
//!
|
||||
//! # Architecture
|
||||
//!
|
||||
//! ```text
|
||||
//! [WASM Channel] → emit-message { attachments=[audio bytes] }
|
||||
//! → [Host: EmittedMessage → IncomingMessage]
|
||||
//! → [TranscriptionMiddleware: detect audio, call provider, replace content]
|
||||
//! → [Agent Loop: sees plain text]
|
||||
//! ```
|
||||
|
||||
pub mod openai;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::channels::{AttachmentKind, IncomingMessage};
|
||||
|
||||
/// Supported audio formats for transcription.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum AudioFormat {
|
||||
/// OGG with Opus codec (Telegram voice notes).
|
||||
OggOpus,
|
||||
/// MP3.
|
||||
Mp3,
|
||||
/// WAV.
|
||||
Wav,
|
||||
/// WebM.
|
||||
Webm,
|
||||
/// M4A / AAC.
|
||||
M4a,
|
||||
}
|
||||
|
||||
impl AudioFormat {
|
||||
/// File extension for this format.
|
||||
pub fn extension(&self) -> &'static str {
|
||||
match self {
|
||||
Self::OggOpus => "ogg",
|
||||
Self::Mp3 => "mp3",
|
||||
Self::Wav => "wav",
|
||||
Self::Webm => "webm",
|
||||
Self::M4a => "m4a",
|
||||
}
|
||||
}
|
||||
|
||||
/// MIME type for this format.
|
||||
pub fn mime_type(&self) -> &'static str {
|
||||
match self {
|
||||
Self::OggOpus => "audio/ogg",
|
||||
Self::Mp3 => "audio/mpeg",
|
||||
Self::Wav => "audio/wav",
|
||||
Self::Webm => "audio/webm",
|
||||
Self::M4a => "audio/mp4",
|
||||
}
|
||||
}
|
||||
|
||||
/// Detect format from MIME type string.
|
||||
pub fn from_mime_type(mime: &str) -> Option<Self> {
|
||||
// Normalize: strip parameters (e.g., "audio/ogg; codecs=opus" → "audio/ogg")
|
||||
let base = mime.split(';').next().unwrap_or(mime).trim();
|
||||
match base {
|
||||
"audio/ogg" | "audio/opus" => Some(Self::OggOpus),
|
||||
"audio/mpeg" | "audio/mp3" => Some(Self::Mp3),
|
||||
"audio/wav" | "audio/x-wav" | "audio/wave" => Some(Self::Wav),
|
||||
"audio/webm" => Some(Self::Webm),
|
||||
"audio/mp4" | "audio/m4a" | "audio/x-m4a" | "audio/aac" => Some(Self::M4a),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Errors from transcription operations.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum TranscriptionError {
|
||||
/// Unsupported audio format.
|
||||
#[error("unsupported audio format: {mime_type}")]
|
||||
UnsupportedFormat { mime_type: String },
|
||||
|
||||
/// Audio file exceeds the provider's size limit.
|
||||
#[error("audio file too large: {size} bytes (max {max})")]
|
||||
FileTooLarge { size: usize, max: usize },
|
||||
|
||||
/// Provider API returned an error.
|
||||
#[error("transcription API error: {message}")]
|
||||
ApiError { message: String },
|
||||
|
||||
/// Network or HTTP error.
|
||||
#[error("transcription request failed: {0}")]
|
||||
RequestFailed(String),
|
||||
|
||||
/// Provider is not configured.
|
||||
#[error("transcription provider not configured: {reason}")]
|
||||
NotConfigured { reason: String },
|
||||
}
|
||||
|
||||
/// Trait for speech-to-text transcription providers.
|
||||
#[async_trait]
|
||||
pub trait TranscriptionProvider: Send + Sync {
|
||||
/// Provider name (e.g., "openai").
|
||||
fn name(&self) -> &str;
|
||||
|
||||
/// Model name (e.g., "whisper-1").
|
||||
fn model_name(&self) -> &str;
|
||||
|
||||
/// Maximum file size in bytes.
|
||||
fn max_file_size(&self) -> usize;
|
||||
|
||||
/// Supported audio formats.
|
||||
fn supported_formats(&self) -> &[AudioFormat];
|
||||
|
||||
/// Transcribe audio bytes to text.
|
||||
async fn transcribe(
|
||||
&self,
|
||||
audio: &[u8],
|
||||
format: AudioFormat,
|
||||
language: Option<&str>,
|
||||
) -> Result<String, TranscriptionError>;
|
||||
}
|
||||
|
||||
/// Middleware that detects audio attachments and transcribes them.
|
||||
pub struct TranscriptionMiddleware {
|
||||
provider: Arc<dyn TranscriptionProvider>,
|
||||
language: Option<String>,
|
||||
}
|
||||
|
||||
impl TranscriptionMiddleware {
|
||||
/// Create a new transcription middleware.
|
||||
pub fn new(provider: Arc<dyn TranscriptionProvider>, language: Option<String>) -> Self {
|
||||
Self { provider, language }
|
||||
}
|
||||
|
||||
/// Process an incoming message, transcribing any audio attachments.
|
||||
///
|
||||
/// If audio attachments are found and transcription succeeds, the message
|
||||
/// content is replaced with the transcribed text. On failure, a fallback
|
||||
/// message is used.
|
||||
pub async fn process(&self, mut msg: IncomingMessage) -> IncomingMessage {
|
||||
let audio_attachment = msg
|
||||
.attachments
|
||||
.iter()
|
||||
.find(|a| a.kind == AttachmentKind::Audio);
|
||||
|
||||
let Some(attachment) = audio_attachment else {
|
||||
return msg;
|
||||
};
|
||||
|
||||
let format = match AudioFormat::from_mime_type(&attachment.mime_type) {
|
||||
Some(f) => f,
|
||||
None => {
|
||||
tracing::warn!(
|
||||
mime = %attachment.mime_type,
|
||||
"Unsupported audio format for transcription"
|
||||
);
|
||||
if msg.content.is_empty() || msg.content == "[Voice note]" {
|
||||
msg.content = "[Voice note: unsupported audio format]".to_string();
|
||||
}
|
||||
return msg;
|
||||
}
|
||||
};
|
||||
|
||||
// Check size limit
|
||||
if attachment.data.len() > self.provider.max_file_size() {
|
||||
tracing::warn!(
|
||||
size = attachment.data.len(),
|
||||
max = self.provider.max_file_size(),
|
||||
"Audio attachment exceeds provider size limit"
|
||||
);
|
||||
if msg.content.is_empty() || msg.content == "[Voice note]" {
|
||||
msg.content = "[Voice note: file too large for transcription]".to_string();
|
||||
}
|
||||
return msg;
|
||||
}
|
||||
|
||||
match self
|
||||
.provider
|
||||
.transcribe(&attachment.data, format, self.language.as_deref())
|
||||
.await
|
||||
{
|
||||
Ok(text) => {
|
||||
tracing::info!(
|
||||
provider = %self.provider.name(),
|
||||
text_len = text.len(),
|
||||
"Audio transcription successful"
|
||||
);
|
||||
msg.content = text;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(
|
||||
error = %e,
|
||||
provider = %self.provider.name(),
|
||||
"Audio transcription failed"
|
||||
);
|
||||
if msg.content.is_empty() || msg.content == "[Voice note]" {
|
||||
msg.content = "[Voice note: transcription failed]".to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
msg
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::channels::Attachment;
|
||||
|
||||
#[test]
|
||||
fn audio_format_from_mime_type() {
|
||||
assert_eq!(
|
||||
AudioFormat::from_mime_type("audio/ogg"),
|
||||
Some(AudioFormat::OggOpus)
|
||||
);
|
||||
assert_eq!(
|
||||
AudioFormat::from_mime_type("audio/ogg; codecs=opus"),
|
||||
Some(AudioFormat::OggOpus)
|
||||
);
|
||||
assert_eq!(
|
||||
AudioFormat::from_mime_type("audio/mpeg"),
|
||||
Some(AudioFormat::Mp3)
|
||||
);
|
||||
assert_eq!(
|
||||
AudioFormat::from_mime_type("audio/mp3"),
|
||||
Some(AudioFormat::Mp3)
|
||||
);
|
||||
assert_eq!(
|
||||
AudioFormat::from_mime_type("audio/wav"),
|
||||
Some(AudioFormat::Wav)
|
||||
);
|
||||
assert_eq!(
|
||||
AudioFormat::from_mime_type("audio/webm"),
|
||||
Some(AudioFormat::Webm)
|
||||
);
|
||||
assert_eq!(
|
||||
AudioFormat::from_mime_type("audio/mp4"),
|
||||
Some(AudioFormat::M4a)
|
||||
);
|
||||
assert_eq!(
|
||||
AudioFormat::from_mime_type("audio/m4a"),
|
||||
Some(AudioFormat::M4a)
|
||||
);
|
||||
assert_eq!(AudioFormat::from_mime_type("text/plain"), None);
|
||||
assert_eq!(AudioFormat::from_mime_type(""), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_format_extension() {
|
||||
assert_eq!(AudioFormat::OggOpus.extension(), "ogg");
|
||||
assert_eq!(AudioFormat::Mp3.extension(), "mp3");
|
||||
assert_eq!(AudioFormat::Wav.extension(), "wav");
|
||||
assert_eq!(AudioFormat::Webm.extension(), "webm");
|
||||
assert_eq!(AudioFormat::M4a.extension(), "m4a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn audio_format_mime_type() {
|
||||
assert_eq!(AudioFormat::OggOpus.mime_type(), "audio/ogg");
|
||||
assert_eq!(AudioFormat::Mp3.mime_type(), "audio/mpeg");
|
||||
}
|
||||
|
||||
/// Mock provider for testing middleware.
|
||||
struct MockProvider {
|
||||
result: Result<String, TranscriptionError>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TranscriptionProvider for MockProvider {
|
||||
fn name(&self) -> &str {
|
||||
"mock"
|
||||
}
|
||||
fn model_name(&self) -> &str {
|
||||
"mock-1"
|
||||
}
|
||||
fn max_file_size(&self) -> usize {
|
||||
25 * 1024 * 1024
|
||||
}
|
||||
fn supported_formats(&self) -> &[AudioFormat] {
|
||||
&[AudioFormat::OggOpus, AudioFormat::Mp3]
|
||||
}
|
||||
async fn transcribe(
|
||||
&self,
|
||||
_audio: &[u8],
|
||||
_format: AudioFormat,
|
||||
_language: Option<&str>,
|
||||
) -> Result<String, TranscriptionError> {
|
||||
match &self.result {
|
||||
Ok(text) => Ok(text.clone()),
|
||||
Err(_) => Err(TranscriptionError::ApiError {
|
||||
message: "mock error".to_string(),
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn middleware_transcribes_audio_attachment() {
|
||||
let provider = Arc::new(MockProvider {
|
||||
result: Ok("Hello, world!".to_string()),
|
||||
});
|
||||
let middleware = TranscriptionMiddleware::new(provider, None);
|
||||
|
||||
let msg = IncomingMessage::new("telegram", "user1", "[Voice note]").with_attachments(vec![
|
||||
Attachment {
|
||||
kind: AttachmentKind::Audio,
|
||||
mime_type: "audio/ogg".to_string(),
|
||||
data: vec![0u8; 100],
|
||||
filename: None,
|
||||
duration_secs: Some(5),
|
||||
},
|
||||
]);
|
||||
|
||||
let result = middleware.process(msg).await;
|
||||
assert_eq!(result.content, "Hello, world!");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn middleware_skips_non_audio_messages() {
|
||||
let provider = Arc::new(MockProvider {
|
||||
result: Ok("transcribed".to_string()),
|
||||
});
|
||||
let middleware = TranscriptionMiddleware::new(provider, None);
|
||||
|
||||
let msg = IncomingMessage::new("telegram", "user1", "regular text");
|
||||
let result = middleware.process(msg).await;
|
||||
assert_eq!(result.content, "regular text");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn middleware_handles_transcription_failure() {
|
||||
let provider = Arc::new(MockProvider {
|
||||
result: Err(TranscriptionError::ApiError {
|
||||
message: "test".to_string(),
|
||||
}),
|
||||
});
|
||||
let middleware = TranscriptionMiddleware::new(provider, None);
|
||||
|
||||
let msg = IncomingMessage::new("telegram", "user1", "[Voice note]").with_attachments(vec![
|
||||
Attachment {
|
||||
kind: AttachmentKind::Audio,
|
||||
mime_type: "audio/ogg".to_string(),
|
||||
data: vec![0u8; 100],
|
||||
filename: None,
|
||||
duration_secs: Some(5),
|
||||
},
|
||||
]);
|
||||
|
||||
let result = middleware.process(msg).await;
|
||||
assert_eq!(result.content, "[Voice note: transcription failed]");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn middleware_handles_unsupported_format() {
|
||||
let provider = Arc::new(MockProvider {
|
||||
result: Ok("text".to_string()),
|
||||
});
|
||||
let middleware = TranscriptionMiddleware::new(provider, None);
|
||||
|
||||
let msg = IncomingMessage::new("telegram", "user1", "[Voice note]").with_attachments(vec![
|
||||
Attachment {
|
||||
kind: AttachmentKind::Audio,
|
||||
mime_type: "audio/flac".to_string(),
|
||||
data: vec![0u8; 100],
|
||||
filename: None,
|
||||
duration_secs: None,
|
||||
},
|
||||
]);
|
||||
|
||||
let result = middleware.process(msg).await;
|
||||
assert_eq!(result.content, "[Voice note: unsupported audio format]");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
//! OpenAI Whisper transcription provider.
|
||||
//!
|
||||
//! Uses the OpenAI `/v1/audio/transcriptions` endpoint with multipart form upload.
|
||||
|
||||
use async_trait::async_trait;
|
||||
use reqwest::multipart;
|
||||
use secrecy::{ExposeSecret, SecretString};
|
||||
|
||||
use crate::transcription::{AudioFormat, TranscriptionError, TranscriptionProvider};
|
||||
|
||||
/// Maximum file size for Whisper API (25 MB).
|
||||
const WHISPER_MAX_FILE_SIZE: usize = 25 * 1024 * 1024;
|
||||
|
||||
/// Supported formats for Whisper.
|
||||
const WHISPER_FORMATS: &[AudioFormat] = &[
|
||||
AudioFormat::OggOpus,
|
||||
AudioFormat::Mp3,
|
||||
AudioFormat::Wav,
|
||||
AudioFormat::Webm,
|
||||
AudioFormat::M4a,
|
||||
];
|
||||
|
||||
/// OpenAI Whisper transcription provider.
|
||||
pub struct OpenAiWhisper {
|
||||
api_key: SecretString,
|
||||
model: String,
|
||||
base_url: String,
|
||||
client: reqwest::Client,
|
||||
}
|
||||
|
||||
impl OpenAiWhisper {
|
||||
/// Create a new OpenAI Whisper provider.
|
||||
pub fn new(api_key: SecretString, model: String) -> Self {
|
||||
Self {
|
||||
api_key,
|
||||
model,
|
||||
base_url: "https://api.openai.com".to_string(),
|
||||
client: reqwest::Client::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set a custom base URL (for testing or alternative endpoints).
|
||||
pub fn with_base_url(mut self, url: impl Into<String>) -> Self {
|
||||
self.base_url = url.into();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl TranscriptionProvider for OpenAiWhisper {
|
||||
fn name(&self) -> &str {
|
||||
"openai"
|
||||
}
|
||||
|
||||
fn model_name(&self) -> &str {
|
||||
&self.model
|
||||
}
|
||||
|
||||
fn max_file_size(&self) -> usize {
|
||||
WHISPER_MAX_FILE_SIZE
|
||||
}
|
||||
|
||||
fn supported_formats(&self) -> &[AudioFormat] {
|
||||
WHISPER_FORMATS
|
||||
}
|
||||
|
||||
async fn transcribe(
|
||||
&self,
|
||||
audio: &[u8],
|
||||
format: AudioFormat,
|
||||
language: Option<&str>,
|
||||
) -> Result<String, TranscriptionError> {
|
||||
if audio.len() > WHISPER_MAX_FILE_SIZE {
|
||||
return Err(TranscriptionError::FileTooLarge {
|
||||
size: audio.len(),
|
||||
max: WHISPER_MAX_FILE_SIZE,
|
||||
});
|
||||
}
|
||||
|
||||
let filename = format!("audio.{}", format.extension());
|
||||
|
||||
// Note: to_vec() copies the audio bytes for the multipart body.
|
||||
// Peak memory is ~2x the file size (original slice + copy). Acceptable
|
||||
// for the 25 MB Whisper limit; revisit if supporting larger files.
|
||||
let file_part = multipart::Part::bytes(audio.to_vec())
|
||||
.file_name(filename)
|
||||
.mime_str(format.mime_type())
|
||||
.map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?;
|
||||
|
||||
let mut form = multipart::Form::new()
|
||||
.part("file", file_part)
|
||||
.text("model", self.model.clone())
|
||||
.text("response_format", "text");
|
||||
|
||||
if let Some(lang) = language {
|
||||
form = form.text("language", lang.to_string());
|
||||
}
|
||||
|
||||
let url = format!("{}/v1/audio/transcriptions", self.base_url);
|
||||
|
||||
let response = self
|
||||
.client
|
||||
.post(&url)
|
||||
.header(
|
||||
"Authorization",
|
||||
format!("Bearer {}", self.api_key.expose_secret()),
|
||||
)
|
||||
.multipart(form)
|
||||
.send()
|
||||
.await
|
||||
.map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?;
|
||||
|
||||
let status = response.status();
|
||||
let body = response
|
||||
.text()
|
||||
.await
|
||||
.map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?;
|
||||
|
||||
if !status.is_success() {
|
||||
return Err(TranscriptionError::ApiError {
|
||||
message: format!("HTTP {}: {}", status, body),
|
||||
});
|
||||
}
|
||||
|
||||
// response_format=text returns raw text, trim whitespace
|
||||
Ok(body.trim().to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn file_too_large_is_rejected_sync() {
|
||||
// The size check happens before any async work, so we can verify
|
||||
// via a simple construction + manual call of the guard logic.
|
||||
let oversized = vec![0u8; WHISPER_MAX_FILE_SIZE + 1];
|
||||
assert!(oversized.len() > WHISPER_MAX_FILE_SIZE);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn file_too_large_returns_error() {
|
||||
let provider = OpenAiWhisper::new(
|
||||
SecretString::from("sk-test".to_string()),
|
||||
"whisper-1".to_string(),
|
||||
);
|
||||
|
||||
let oversized = vec![0u8; WHISPER_MAX_FILE_SIZE + 1];
|
||||
let result = provider
|
||||
.transcribe(&oversized, AudioFormat::Wav, None)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Err(TranscriptionError::FileTooLarge { size, max }) => {
|
||||
assert_eq!(size, WHISPER_MAX_FILE_SIZE + 1);
|
||||
assert_eq!(max, WHISPER_MAX_FILE_SIZE);
|
||||
}
|
||||
other => panic!("Expected FileTooLarge, got: {:?}", other),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn api_error_on_bad_url() {
|
||||
// Point at a URL that will fail to connect
|
||||
let provider = OpenAiWhisper::new(
|
||||
SecretString::from("sk-test".to_string()),
|
||||
"whisper-1".to_string(),
|
||||
)
|
||||
.with_base_url("http://127.0.0.1:1"); // port 1 won't be listening
|
||||
|
||||
let audio = vec![0u8; 100];
|
||||
let result = provider
|
||||
.transcribe(&audio, AudioFormat::OggOpus, Some("en"))
|
||||
.await;
|
||||
|
||||
assert!(
|
||||
matches!(result, Err(TranscriptionError::RequestFailed(_))),
|
||||
"Expected RequestFailed, got: {:?}",
|
||||
result
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_metadata() {
|
||||
let provider = OpenAiWhisper::new(
|
||||
SecretString::from("sk-test".to_string()),
|
||||
"whisper-1".to_string(),
|
||||
);
|
||||
|
||||
assert_eq!(provider.name(), "openai");
|
||||
assert_eq!(provider.model_name(), "whisper-1");
|
||||
assert_eq!(provider.max_file_size(), WHISPER_MAX_FILE_SIZE);
|
||||
assert_eq!(provider.supported_formats().len(), 5);
|
||||
}
|
||||
}
|
||||
@@ -113,6 +113,27 @@ interface channel-host {
|
||||
|
||||
// ==================== Channel-Specific Capabilities ====================
|
||||
|
||||
/// Kind of attachment (audio, image, document).
|
||||
enum attachment-kind {
|
||||
audio,
|
||||
image,
|
||||
document,
|
||||
}
|
||||
|
||||
/// Binary attachment on an emitted message (e.g., voice note, photo).
|
||||
record attachment {
|
||||
/// What kind of content this is.
|
||||
kind: attachment-kind,
|
||||
/// MIME type (e.g., "audio/ogg", "image/jpeg").
|
||||
mime-type: string,
|
||||
/// Raw bytes of the attachment.
|
||||
data: list<u8>,
|
||||
/// Optional filename.
|
||||
filename: option<string>,
|
||||
/// Duration in seconds (for audio/video).
|
||||
duration-secs: option<u32>,
|
||||
}
|
||||
|
||||
/// A message to emit to the agent.
|
||||
record emitted-message {
|
||||
/// User identifier within the channel (e.g., Slack user ID).
|
||||
@@ -125,6 +146,8 @@ interface channel-host {
|
||||
thread-id: option<string>,
|
||||
/// Channel-specific metadata as JSON string.
|
||||
metadata-json: string,
|
||||
/// Optional binary attachments (voice notes, images, etc.).
|
||||
attachments: list<attachment>,
|
||||
}
|
||||
|
||||
/// Emit a message to the agent.
|
||||
|
||||
Reference in New Issue
Block a user