From 877f117096800349bc8d0bb053331720618e8d4e Mon Sep 17 00:00:00 2001 From: smkrv <17809065+smkrv@users.noreply.github.com> Date: Mon, 16 Mar 2026 10:57:58 +0300 Subject: [PATCH] feat(transcription): add Chat Completions API provider for audio transcription (#1130) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(transcription): add Chat Completions API provider for audio transcription The existing transcription pipeline only supports the OpenAI Whisper API (/v1/audio/transcriptions with multipart upload). Providers like OpenRouter expose audio transcription through the Chat Completions API instead, using base64-encoded audio in the `input_audio` content type. Add `ChatCompletionsTranscriptionProvider` that sends audio as base64 in a chat completion request and extracts the transcript from the response. Compatible with OpenRouter, OpenAI GPT-4o-audio, and any provider that supports audio input via Chat Completions. Config changes: - TRANSCRIPTION_PROVIDER=chat_completions selects the new provider - TRANSCRIPTION_API_KEY overrides provider-specific keys - LLM_API_KEY used as fallback for chat_completions provider - Default model per provider (whisper-1 for openai, gemini-2.0-flash for chat_completions) * style: address review feedback — formatting, idiomatic patterns - Fix rustfmt formatting for provider constructor chain - Use or_else for resolve_api_key priority chain (Gemini review) - Use trim_end_matches('/') instead of while loop (Gemini review) --------- Co-authored-by: SMKRV --- src/config/transcription.rs | 77 +++++++++-- src/transcription/chat_completions.rs | 179 ++++++++++++++++++++++++++ src/transcription/mod.rs | 2 + 3 files changed, 245 insertions(+), 13 deletions(-) create mode 100644 src/transcription/chat_completions.rs diff --git a/src/config/transcription.rs b/src/config/transcription.rs index b0f76066..da2bac25 100644 --- a/src/config/transcription.rs +++ b/src/config/transcription.rs @@ -9,11 +9,15 @@ use crate::settings::Settings; pub struct TranscriptionConfig { /// Whether audio transcription is enabled. pub enabled: bool, - /// Provider: "openai" (default). + /// Provider: "openai" (default) or "chat_completions". pub provider: String, /// OpenAI API key (reuses OPENAI_API_KEY). pub openai_api_key: Option, - /// Model to use (default: "whisper-1"). + /// Explicit transcription API key (overrides provider-specific keys). + pub api_key: Option, + /// LLM API key (reuses LLM_API_KEY, used as fallback for chat_completions). + pub llm_api_key: Option, + /// Model to use (default depends on provider). pub model: String, /// Base URL override for the transcription API. pub base_url: Option, @@ -25,6 +29,8 @@ impl Default for TranscriptionConfig { enabled: false, provider: "openai".to_string(), openai_api_key: None, + api_key: None, + llm_api_key: None, model: "whisper-1".to_string(), base_url: None, } @@ -42,8 +48,15 @@ impl TranscriptionConfig { optional_env("TRANSCRIPTION_PROVIDER")?.unwrap_or_else(|| "openai".to_string()); let openai_api_key = optional_env("OPENAI_API_KEY")?.map(SecretString::from); + let api_key = optional_env("TRANSCRIPTION_API_KEY")?.map(SecretString::from); + let llm_api_key = optional_env("LLM_API_KEY")?.map(SecretString::from); - let model = optional_env("TRANSCRIPTION_MODEL")?.unwrap_or_else(|| "whisper-1".to_string()); + let default_model = match provider.as_str() { + "chat_completions" => "google/gemini-2.0-flash-001", + _ => "whisper-1", + }; + let model = + optional_env("TRANSCRIPTION_MODEL")?.unwrap_or_else(|| default_model.to_string()); let base_url = optional_env("TRANSCRIPTION_BASE_URL")?; @@ -51,29 +64,67 @@ impl TranscriptionConfig { enabled, provider, openai_api_key, + api_key, + llm_api_key, model, base_url, }) } + /// Resolve the API key for the configured provider. + /// + /// Priority: `TRANSCRIPTION_API_KEY` > provider-specific key. + fn resolve_api_key(&self) -> Option<&SecretString> { + self.api_key + .as_ref() + .or_else(|| match self.provider.as_str() { + "chat_completions" => self.llm_api_key.as_ref().or(self.openai_api_key.as_ref()), + _ => self.openai_api_key.as_ref(), + }) + } + /// Create the transcription provider if enabled and configured. pub fn create_provider(&self) -> Option> { if !self.enabled { return None; } - // Currently only OpenAI Whisper is supported; more providers can be - // added here with a match on self.provider. - let api_key = self.openai_api_key.as_ref()?; - tracing::info!(model = %self.model, "Audio transcription enabled via OpenAI Whisper"); + let api_key = self.resolve_api_key()?; - let mut provider = crate::transcription::OpenAiWhisperProvider::new(api_key.clone()) - .with_model(&self.model); + match self.provider.as_str() { + "chat_completions" => { + tracing::info!( + model = %self.model, + "Audio transcription enabled via Chat Completions API" + ); - if let Some(ref base_url) = self.base_url { - provider = provider.with_base_url(base_url); + let mut provider = crate::transcription::ChatCompletionsTranscriptionProvider::new( + api_key.clone(), + ) + .with_model(&self.model); + + if let Some(ref base_url) = self.base_url { + provider = provider.with_base_url(base_url); + } + + Some(Box::new(provider)) + } + _ => { + tracing::info!( + model = %self.model, + "Audio transcription enabled via OpenAI Whisper" + ); + + let mut provider = + crate::transcription::OpenAiWhisperProvider::new(api_key.clone()) + .with_model(&self.model); + + if let Some(ref base_url) = self.base_url { + provider = provider.with_base_url(base_url); + } + + Some(Box::new(provider)) + } } - - Some(Box::new(provider)) } } diff --git a/src/transcription/chat_completions.rs b/src/transcription/chat_completions.rs new file mode 100644 index 00000000..e23818aa --- /dev/null +++ b/src/transcription/chat_completions.rs @@ -0,0 +1,179 @@ +//! Chat Completions-based transcription provider. +//! +//! Uses the `/v1/chat/completions` endpoint with `input_audio` content type +//! to transcribe audio. Compatible with OpenRouter, OpenAI GPT-4o-audio, and +//! any provider that supports audio input via the Chat Completions API. + +use async_trait::async_trait; +use base64::Engine; +use secrecy::{ExposeSecret, SecretString}; + +use super::{AudioFormat, TranscriptionError, TranscriptionProvider}; + +/// Transcription provider that sends audio via the Chat Completions API. +/// +/// Unlike the Whisper provider (which uses `/v1/audio/transcriptions` with +/// multipart upload), this provider sends base64-encoded audio as an +/// `input_audio` content part in a chat message, enabling use with +/// OpenRouter and other providers that only expose audio through the +/// Chat Completions API. +pub struct ChatCompletionsTranscriptionProvider { + client: reqwest::Client, + api_key: SecretString, + model: String, + base_url: String, +} + +impl ChatCompletionsTranscriptionProvider { + /// Create a new provider with the given API key. + pub fn new(api_key: SecretString) -> Self { + Self { + client: match reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(120)) + .build() + { + Ok(c) => c, + Err(e) => { + tracing::error!( + "Failed to build HTTP client with timeout, falling back to default: {e}" + ); + reqwest::Client::default() + } + }, + api_key, + model: "google/gemini-2.0-flash-001".to_string(), + base_url: "https://openrouter.ai/api".to_string(), + } + } + + /// Override the base URL. + pub fn with_base_url(mut self, base_url: impl Into) -> Self { + self.base_url = base_url.into().trim_end_matches('/').to_string(); + self + } + + /// Override the model name. + pub fn with_model(mut self, model: impl Into) -> Self { + self.model = model.into(); + self + } +} + +/// Map [`AudioFormat`] to the format string expected by the Chat Completions API. +fn audio_format_str(format: AudioFormat) -> &'static str { + match format { + AudioFormat::Ogg => "ogg", + AudioFormat::Mp3 => "mp3", + AudioFormat::Mp4 => "mp4", + AudioFormat::Wav => "wav", + AudioFormat::Webm => "webm", + AudioFormat::Flac => "flac", + AudioFormat::M4a => "m4a", + } +} + +#[async_trait] +impl TranscriptionProvider for ChatCompletionsTranscriptionProvider { + async fn transcribe( + &self, + audio_data: &[u8], + format: AudioFormat, + ) -> Result { + if audio_data.is_empty() { + return Err(TranscriptionError::EmptyAudio); + } + + let b64 = base64::engine::general_purpose::STANDARD.encode(audio_data); + + let body = serde_json::json!({ + "model": self.model, + "messages": [{ + "role": "user", + "content": [ + { + "type": "text", + "text": "Transcribe this audio. Return only the transcript text, nothing else." + }, + { + "type": "input_audio", + "input_audio": { + "data": b64, + "format": audio_format_str(format) + } + } + ] + }] + }); + + let url = format!("{}/v1/chat/completions", self.base_url); + + let response = self + .client + .post(&url) + .header( + "Authorization", + format!("Bearer {}", self.api_key.expose_secret()), + ) + .json(&body) + .send() + .await + .map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?; + + let status = response.status(); + if !status.is_success() { + let body = response + .text() + .await + .unwrap_or_else(|_| "unknown error".to_string()); + return Err(TranscriptionError::RequestFailed(format!( + "HTTP {}: {}", + status, body + ))); + } + + let json: serde_json::Value = response + .json() + .await + .map_err(|e| TranscriptionError::RequestFailed(e.to_string()))?; + + // Extract text from the standard Chat Completions response format: + // { "choices": [{ "message": { "content": "..." } }] } + let text = json + .get("choices") + .and_then(|c| c.get(0)) + .and_then(|c| c.get("message")) + .and_then(|m| m.get("content")) + .and_then(|c| c.as_str()) + .ok_or_else(|| { + TranscriptionError::RequestFailed( + "unexpected response format: missing choices[0].message.content".to_string(), + ) + })?; + + Ok(text.trim().to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn audio_format_str_maps_all_variants() { + assert_eq!(audio_format_str(AudioFormat::Ogg), "ogg"); + assert_eq!(audio_format_str(AudioFormat::Mp3), "mp3"); + assert_eq!(audio_format_str(AudioFormat::Mp4), "mp4"); + assert_eq!(audio_format_str(AudioFormat::Wav), "wav"); + assert_eq!(audio_format_str(AudioFormat::Webm), "webm"); + assert_eq!(audio_format_str(AudioFormat::Flac), "flac"); + assert_eq!(audio_format_str(AudioFormat::M4a), "m4a"); + } + + #[tokio::test] + async fn rejects_empty_audio() { + let provider = + ChatCompletionsTranscriptionProvider::new(SecretString::from("test-key".to_string())); + let result = provider.transcribe(&[], AudioFormat::Ogg).await; + assert!(matches!(result, Err(TranscriptionError::EmptyAudio))); + } +} diff --git a/src/transcription/mod.rs b/src/transcription/mod.rs index d0a7d31c..ab2e43f9 100644 --- a/src/transcription/mod.rs +++ b/src/transcription/mod.rs @@ -4,8 +4,10 @@ //! backends and a [`TranscriptionMiddleware`] that detects audio attachments //! on incoming messages and replaces them with transcribed text. +mod chat_completions; mod openai; +pub use self::chat_completions::ChatCompletionsTranscriptionProvider; pub use self::openai::OpenAiWhisperProvider; use async_trait::async_trait;