refactor(llm): move transcription module into src/llm/ (#1559)

* refactor(llm): move transcription module into src/llm/ Transcription is an LLM capability (Whisper, Chat Completions audio). Move it from a top-level module into src/llm/transcription/ to reflect this, and update all references across the codebase. Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]> * style: fix rustfmt formatting after module move Co-Authored-By: Claude Opus 4.6 (1M context) <[email protected]> --------- Co-authored-by: Claude Opus 4.6 (1M context) <[email protected]>
2026-08-30 16:19:21 +00:00 · 2026-03-22 00:25:54 -07:00
parent 1a62febe67
commit fbce9a5fe3
8 changed files with 16 additions and 12 deletions
@@ -35,6 +35,7 @@ mod rig_adapter;
 pub mod session;
 pub mod smart_routing;
 mod token_refreshing;
+pub mod transcription;

 #[cfg(test)]
 mod codex_test_helpers;
@@ -0,0 +1,179 @@
+//! Chat Completions-based transcription provider.
+//!
+//! Uses the `/v1/chat/completions` endpoint with `input_audio` content type
+//! to transcribe audio. Compatible with OpenRouter, OpenAI GPT-4o-audio, and
+//! any provider that supports audio input via the Chat Completions API.
+
+use async_trait::async_trait;
+use base64::Engine;
+use secrecy::{ExposeSecret, SecretString};
+
+use super::{AudioFormat, TranscriptionError, TranscriptionProvider};
+
+/// Transcription provider that sends audio via the Chat Completions API.
+///
+/// Unlike the Whisper provider (which uses `/v1/audio/transcriptions` with
+/// multipart upload), this provider sends base64-encoded audio as an
+/// `input_audio` content part in a chat message, enabling use with
+/// OpenRouter and other providers that only expose audio through the
+/// Chat Completions API.
+pub struct ChatCompletionsTranscriptionProvider {
+    client: reqwest::Client,
+    api_key: SecretString,
+    model: String,
+    base_url: String,
+}
+
+impl ChatCompletionsTranscriptionProvider {
+    /// Create a new provider with the given API key.
+    pub fn new(api_key: SecretString) -> Self {
+        Self {
+            client: match reqwest::Client::builder()
+                .timeout(std::time::Duration::from_secs(120))
+                .build()
+            {
+                Ok(c) => c,
+                Err(e) => {
+                    tracing::error!(
+                        "Failed to build HTTP client with timeout, falling back to default: {e}"
+                    );
+                    reqwest::Client::default()
+                }
+            },
+            api_key,
+            model: "google/gemini-2.0-flash-001".to_string(),
+            base_url: "https://openrouter.ai/api".to_string(),
+        }
+    }
+
+    /// Override the base URL.
+    pub fn with_base_url(mut self, base_url: impl Into<String>) -> Self {
+        self.base_url = base_url.into().trim_end_matches('/').to_string();
+        self
+    }
+
+    /// Override the model name.
+    pub fn with_model(mut self, model: impl Into<String>) -> Self {
+        self.model = model.into();
+        self
+    }
+}
+
+/// Map [`AudioFormat`] to the format string expected by the Chat Completions API.
+fn audio_format_str(format: AudioFormat) -> &'static str {
+    match format {
+        AudioFormat::Ogg => "ogg",
+        AudioFormat::Mp3 => "mp3",
+        AudioFormat::Mp4 => "mp4",
+        AudioFormat::Wav => "wav",
+        AudioFormat::Webm => "webm",
+        AudioFormat::Flac => "flac",
+        AudioFormat::M4a => "m4a",
+    }
+}
+
+#[async_trait]
+impl TranscriptionProvider for ChatCompletionsTranscriptionProvider {
+    async fn transcribe(
+        &self,
+        audio_data: &[u8],
+        format: AudioFormat,
+    ) -> Result<String, TranscriptionError> {
+        if audio_data.is_empty() {
+            return Err(TranscriptionError::EmptyAudio);
+        }
+
+        let b64 = base64::engine::general_purpose::STANDARD.encode(audio_data);
+
+        let body = serde_json::json!({
+            "model": self.model,
+            "messages": [{
+                "role": "user",
+                "content": [
+                    {
+                        "type": "text",
+                        "text": "Transcribe this audio. Return only the transcript text, nothing else."
+                    },
+                    {
+                        "type": "input_audio",
+                        "input_audio": {
+                            "data": b64,
+                            "format": audio_format_str(format)
+                        }
+                    }
+                ]
+            }]
+        });
+
+        let url = format!("{}/v1/chat/completions", self.base_url);
+
+        let response = self
+            .client
+            .post(&url)
+            .header(
+                "Authorization",
+                format!("Bearer {}", self.api_key.expose_secret()),
+            )
+            .json(&body)
+            .send()
+            .await
+            .map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?;
+
+        let status = response.status();
+        if !status.is_success() {
+            let body = response
+                .text()
+                .await
+                .unwrap_or_else(|_| "unknown error".to_string());
+            return Err(TranscriptionError::RequestFailed(format!(
+                "HTTP {}: {}",
+                status, body
+            )));
+        }
+
+        let json: serde_json::Value = response
+            .json()
+            .await
+            .map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?;
+
+        // Extract text from the standard Chat Completions response format:
+        // { "choices": [{ "message": { "content": "..." } }] }
+        let text = json
+            .get("choices")
+            .and_then(|c| c.get(0))
+            .and_then(|c| c.get("message"))
+            .and_then(|m| m.get("content"))
+            .and_then(|c| c.as_str())
+            .ok_or_else(|| {
+                TranscriptionError::RequestFailed(
+                    "unexpected response format: missing choices[0].message.content".to_string(),
+                )
+            })?;
+
+        Ok(text.trim().to_string())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn audio_format_str_maps_all_variants() {
+        assert_eq!(audio_format_str(AudioFormat::Ogg), "ogg");
+        assert_eq!(audio_format_str(AudioFormat::Mp3), "mp3");
+        assert_eq!(audio_format_str(AudioFormat::Mp4), "mp4");
+        assert_eq!(audio_format_str(AudioFormat::Wav), "wav");
+        assert_eq!(audio_format_str(AudioFormat::Webm), "webm");
+        assert_eq!(audio_format_str(AudioFormat::Flac), "flac");
+        assert_eq!(audio_format_str(AudioFormat::M4a), "m4a");
+    }
+
+    #[tokio::test]
+    async fn rejects_empty_audio() {
+        let provider =
+            ChatCompletionsTranscriptionProvider::new(SecretString::from("test-key".to_string()));
+        let result = provider.transcribe(&[], AudioFormat::Ogg).await;
+        assert!(matches!(result, Err(TranscriptionError::EmptyAudio)));
+    }
+}
@@ -0,0 +1,289 @@
+//! Audio transcription pipeline.
+//!
+//! Provides a [`TranscriptionProvider`] trait for pluggable speech-to-text
+//! backends and a [`TranscriptionMiddleware`] that detects audio attachments
+//! on incoming messages and replaces them with transcribed text.
+
+mod chat_completions;
+mod openai;
+
+pub use self::chat_completions::ChatCompletionsTranscriptionProvider;
+pub use self::openai::OpenAiWhisperProvider;
+
+use async_trait::async_trait;
+
+/// Supported audio formats for transcription.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum AudioFormat {
+    Ogg,
+    Mp3,
+    Mp4,
+    Wav,
+    Webm,
+    Flac,
+    M4a,
+}
+
+impl AudioFormat {
+    /// Infer audio format from MIME type. Returns `None` for unsupported types.
+    pub fn from_mime_type(mime: &str) -> Option<Self> {
+        let base = mime.split(';').next().unwrap_or(mime).trim();
+        match base {
+            "audio/ogg" | "audio/opus" => Some(Self::Ogg),
+            "audio/mpeg" | "audio/mp3" => Some(Self::Mp3),
+            "audio/mp4" => Some(Self::Mp4),
+            "audio/wav" | "audio/x-wav" => Some(Self::Wav),
+            "audio/webm" => Some(Self::Webm),
+            "audio/flac" | "audio/x-flac" => Some(Self::Flac),
+            "audio/m4a" | "audio/x-m4a" | "audio/aac" => Some(Self::M4a),
+            _ => None,
+        }
+    }
+
+    /// File extension for this format (used as the filename in multipart uploads).
+    pub fn extension(&self) -> &'static str {
+        match self {
+            Self::Ogg => "ogg",
+            Self::Mp3 => "mp3",
+            Self::Mp4 => "mp4",
+            Self::Wav => "wav",
+            Self::Webm => "webm",
+            Self::Flac => "flac",
+            Self::M4a => "m4a",
+        }
+    }
+}
+
+/// Errors from the transcription pipeline.
+#[derive(Debug, thiserror::Error)]
+pub enum TranscriptionError {
+    #[error("Transcription request failed: {0}")]
+    RequestFailed(String),
+
+    #[error("Unsupported audio format: {mime_type}")]
+    UnsupportedFormat { mime_type: String },
+
+    #[error("Audio data is empty")]
+    EmptyAudio,
+}
+
+/// Trait for speech-to-text providers.
+#[async_trait]
+pub trait TranscriptionProvider: Send + Sync {
+    /// Transcribe audio bytes into text.
+    async fn transcribe(
+        &self,
+        audio_data: &[u8],
+        format: AudioFormat,
+    ) -> Result<String, TranscriptionError>;
+}
+
+/// Middleware that processes audio attachments on incoming messages.
+///
+/// When an incoming message has audio attachments with inline data,
+/// the middleware transcribes them and sets `extracted_text` on the attachment.
+/// If the message has no text content, the transcription becomes the message content.
+pub struct TranscriptionMiddleware {
+    provider: Box<dyn TranscriptionProvider>,
+}
+
+impl TranscriptionMiddleware {
+    /// Create a new middleware with the given transcription provider.
+    pub fn new(provider: Box<dyn TranscriptionProvider>) -> Self {
+        Self { provider }
+    }
+
+    /// Process an incoming message, transcribing any audio attachments with data.
+    ///
+    /// Modifies the message in place:
+    /// - Sets `extracted_text` on audio attachments that have inline data
+    /// - If the message content is empty, sets it to the transcription
+    pub async fn process(&self, msg: &mut crate::channels::IncomingMessage) {
+        use crate::channels::AttachmentKind;
+
+        let mut transcriptions = Vec::new();
+
+        for (i, attachment) in msg.attachments.iter().enumerate() {
+            if attachment.kind != AttachmentKind::Audio {
+                continue;
+            }
+            if attachment.data.is_empty() {
+                continue;
+            }
+            // Already transcribed
+            if attachment.extracted_text.is_some() {
+                continue;
+            }
+
+            let format = match AudioFormat::from_mime_type(&attachment.mime_type) {
+                Some(f) => f,
+                None => {
+                    tracing::warn!(
+                        mime = %attachment.mime_type,
+                        "Skipping audio attachment with unsupported format"
+                    );
+                    continue;
+                }
+            };
+
+            match self.provider.transcribe(&attachment.data, format).await {
+                Ok(text) => {
+                    tracing::info!(
+                        attachment_id = %attachment.id,
+                        text_len = text.len(),
+                        "Transcribed audio attachment"
+                    );
+                    transcriptions.push((i, text));
+                }
+                Err(e) => {
+                    tracing::error!(
+                        attachment_id = %attachment.id,
+                        error = %e,
+                        "Failed to transcribe audio attachment"
+                    );
+                    transcriptions.push((i, format!("[Transcription failed: {}]", e)));
+                }
+            }
+        }
+
+        for (i, text) in &transcriptions {
+            msg.attachments[*i].extracted_text = Some(text.clone());
+        }
+
+        // If message has no text content, use the first successful transcription
+        if (msg.content.is_empty() || msg.content == "[Voice note]")
+            && let Some((_, text)) = transcriptions
+                .iter()
+                .find(|(_, t)| !t.starts_with("[Transcription failed"))
+        {
+            msg.content = text.clone();
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::channels::{AttachmentKind, IncomingAttachment, IncomingMessage};
+
+    struct MockProvider {
+        result: Result<String, TranscriptionError>,
+    }
+
+    #[async_trait]
+    impl TranscriptionProvider for MockProvider {
+        async fn transcribe(
+            &self,
+            _audio_data: &[u8],
+            _format: AudioFormat,
+        ) -> Result<String, TranscriptionError> {
+            match &self.result {
+                Ok(text) => Ok(text.clone()),
+                Err(_) => Err(TranscriptionError::RequestFailed("mock error".into())),
+            }
+        }
+    }
+
+    fn voice_attachment(data: Vec<u8>) -> IncomingAttachment {
+        IncomingAttachment {
+            id: "voice_123".to_string(),
+            kind: AttachmentKind::Audio,
+            mime_type: "audio/ogg".to_string(),
+            filename: Some("voice.ogg".to_string()),
+            size_bytes: Some(data.len() as u64),
+            source_url: None,
+            storage_key: None,
+            extracted_text: None,
+            data,
+            duration_secs: Some(5),
+        }
+    }
+
+    #[tokio::test]
+    async fn middleware_transcribes_audio_attachment() {
+        let middleware = TranscriptionMiddleware::new(Box::new(MockProvider {
+            result: Ok("Hello world".to_string()),
+        }));
+
+        let mut msg = IncomingMessage::new("telegram", "user1", "[Voice note]")
+            .with_attachments(vec![voice_attachment(vec![1, 2, 3])]);
+
+        middleware.process(&mut msg).await;
+
+        assert_eq!(
+            msg.attachments[0].extracted_text.as_deref(),
+            Some("Hello world")
+        );
+        assert_eq!(msg.content, "Hello world");
+    }
+
+    #[tokio::test]
+    async fn middleware_skips_empty_audio_data() {
+        let middleware = TranscriptionMiddleware::new(Box::new(MockProvider {
+            result: Ok("Should not be called".to_string()),
+        }));
+
+        let mut msg = IncomingMessage::new("telegram", "user1", "text message")
+            .with_attachments(vec![voice_attachment(Vec::new())]);
+
+        middleware.process(&mut msg).await;
+
+        assert!(msg.attachments[0].extracted_text.is_none());
+        assert_eq!(msg.content, "text message");
+    }
+
+    #[tokio::test]
+    async fn middleware_skips_already_transcribed() {
+        let middleware = TranscriptionMiddleware::new(Box::new(MockProvider {
+            result: Ok("New transcription".to_string()),
+        }));
+
+        let mut attachment = voice_attachment(vec![1, 2, 3]);
+        attachment.extracted_text = Some("Already done".to_string());
+
+        let mut msg =
+            IncomingMessage::new("telegram", "user1", "").with_attachments(vec![attachment]);
+
+        middleware.process(&mut msg).await;
+
+        assert_eq!(
+            msg.attachments[0].extracted_text.as_deref(),
+            Some("Already done")
+        );
+    }
+
+    #[tokio::test]
+    async fn middleware_preserves_existing_content() {
+        let middleware = TranscriptionMiddleware::new(Box::new(MockProvider {
+            result: Ok("Transcription".to_string()),
+        }));
+
+        let mut msg = IncomingMessage::new("telegram", "user1", "User typed this")
+            .with_attachments(vec![voice_attachment(vec![1, 2, 3])]);
+
+        middleware.process(&mut msg).await;
+
+        assert_eq!(
+            msg.attachments[0].extracted_text.as_deref(),
+            Some("Transcription")
+        );
+        assert_eq!(msg.content, "User typed this");
+    }
+
+    #[test]
+    fn audio_format_from_mime() {
+        assert_eq!(
+            AudioFormat::from_mime_type("audio/ogg"),
+            Some(AudioFormat::Ogg)
+        );
+        assert_eq!(
+            AudioFormat::from_mime_type("audio/mpeg"),
+            Some(AudioFormat::Mp3)
+        );
+        assert_eq!(
+            AudioFormat::from_mime_type("audio/ogg; codecs=opus"),
+            Some(AudioFormat::Ogg)
+        );
+        assert_eq!(AudioFormat::from_mime_type("image/jpeg"), None);
+    }
+}
@@ -0,0 +1,124 @@
+//! OpenAI Whisper transcription provider.
+
+use async_trait::async_trait;
+use reqwest::multipart;
+use secrecy::{ExposeSecret, SecretString};
+
+use super::{AudioFormat, TranscriptionError, TranscriptionProvider};
+
+/// OpenAI Whisper speech-to-text provider.
+///
+/// Uses the `/v1/audio/transcriptions` endpoint.
+pub struct OpenAiWhisperProvider {
+    client: reqwest::Client,
+    api_key: SecretString,
+    model: String,
+    base_url: String,
+}
+
+impl OpenAiWhisperProvider {
+    /// Create a new Whisper provider with the given API key.
+    pub fn new(api_key: SecretString) -> Self {
+        Self {
+            client: match reqwest::Client::builder()
+                .timeout(std::time::Duration::from_secs(120))
+                .build()
+            {
+                Ok(c) => c,
+                Err(e) => {
+                    tracing::error!(
+                        "Failed to build HTTP client with timeout, falling back to default: {e}"
+                    );
+                    reqwest::Client::default()
+                }
+            },
+            api_key,
+            model: "whisper-1".to_string(),
+            base_url: "https://api.openai.com".to_string(),
+        }
+    }
+
+    /// Override the base URL (for proxied or compatible endpoints).
+    pub fn with_base_url(mut self, base_url: impl Into<String>) -> Self {
+        let mut url = base_url.into();
+        // Normalize: strip trailing slash to avoid double-slash in URL construction
+        while url.ends_with('/') {
+            url.pop();
+        }
+        self.base_url = url;
+        self
+    }
+
+    /// Override the model name.
+    pub fn with_model(mut self, model: impl Into<String>) -> Self {
+        self.model = model.into();
+        self
+    }
+}
+
+#[async_trait]
+impl TranscriptionProvider for OpenAiWhisperProvider {
+    async fn transcribe(
+        &self,
+        audio_data: &[u8],
+        format: AudioFormat,
+    ) -> Result<String, TranscriptionError> {
+        if audio_data.is_empty() {
+            return Err(TranscriptionError::EmptyAudio);
+        }
+
+        let filename = format!("audio.{}", format.extension());
+        let mime_str = match format {
+            AudioFormat::Ogg => "audio/ogg",
+            AudioFormat::Mp3 => "audio/mpeg",
+            AudioFormat::Mp4 => "audio/mp4",
+            AudioFormat::Wav => "audio/wav",
+            AudioFormat::Webm => "audio/webm",
+            AudioFormat::Flac => "audio/flac",
+            AudioFormat::M4a => "audio/m4a",
+        };
+
+        let file_part = multipart::Part::bytes(audio_data.to_vec())
+            .file_name(filename)
+            .mime_str(mime_str)
+            .map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?;
+
+        let form = multipart::Form::new()
+            .text("model", self.model.clone())
+            .text("response_format", "text")
+            .part("file", file_part);
+
+        let url = format!("{}/v1/audio/transcriptions", self.base_url);
+
+        let response = self
+            .client
+            .post(&url)
+            .header(
+                "Authorization",
+                format!("Bearer {}", self.api_key.expose_secret()),
+            )
+            .multipart(form)
+            .send()
+            .await
+            .map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?;
+
+        let status = response.status();
+        if !status.is_success() {
+            let body = response
+                .text()
+                .await
+                .unwrap_or_else(|_| "unknown error".to_string());
+            return Err(TranscriptionError::RequestFailed(format!(
+                "HTTP {}: {}",
+                status, body
+            )));
+        }
+
+        let text = response
+            .text()
+            .await
+            .map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?;
+
+        Ok(text.trim().to_string())
+    }
+}