mirror of
https://github.com/outbackdingo/optimclaw.git
synced 2026-08-27 08:00:17 +00:00
feat: add audio transcription pipeline with OpenAI Whisper and Telegram voice notes (#90)
Adds speech-to-text support so WASM channels can emit audio attachments that get automatically transcribed before reaching the agent. Telegram voice notes are the first integration — downloaded via Bot API and transcribed via OpenAI Whisper. - Extend WIT with attachment-kind, attachment records on emitted-message - Add Attachment/AttachmentKind types to channel.rs and IncomingMessage - Add TranscriptionProvider trait, AudioFormat enum, TranscriptionMiddleware - Implement OpenAI Whisper provider (multipart POST, 25MB limit) - Add TranscriptionConfig + TranscriptionSettings with env var overrides - Parse Telegram voice messages, download via getFile, emit as attachments - Apply transcription in both process/dispatch emitted message paths - Graceful degradation: download failures show "[Voice note: download failed]" - Validate attachment sizes (10MB max), drop oversized without losing message Co-Authored-By: Claude Opus 4.6 <[email protected]>
This commit is contained in:
co-authored by
Claude Opus 4.6
parent
b68d67bd35
commit
bbb2d5c4dd
@@ -113,6 +113,27 @@ interface channel-host {
|
||||
|
||||
// ==================== Channel-Specific Capabilities ====================
|
||||
|
||||
/// Kind of attachment (audio, image, document).
|
||||
enum attachment-kind {
|
||||
audio,
|
||||
image,
|
||||
document,
|
||||
}
|
||||
|
||||
/// Binary attachment on an emitted message (e.g., voice note, photo).
|
||||
record attachment {
|
||||
/// What kind of content this is.
|
||||
kind: attachment-kind,
|
||||
/// MIME type (e.g., "audio/ogg", "image/jpeg").
|
||||
mime-type: string,
|
||||
/// Raw bytes of the attachment.
|
||||
data: list<u8>,
|
||||
/// Optional filename.
|
||||
filename: option<string>,
|
||||
/// Duration in seconds (for audio/video).
|
||||
duration-secs: option<u32>,
|
||||
}
|
||||
|
||||
/// A message to emit to the agent.
|
||||
record emitted-message {
|
||||
/// User identifier within the channel (e.g., Slack user ID).
|
||||
@@ -125,6 +146,8 @@ interface channel-host {
|
||||
thread-id: option<string>,
|
||||
/// Channel-specific metadata as JSON string.
|
||||
metadata-json: string,
|
||||
/// Optional binary attachments (voice notes, images, etc.).
|
||||
attachments: list<attachment>,
|
||||
}
|
||||
|
||||
/// Emit a message to the agent.
|
||||
|
||||
Reference in New Issue
Block a user