diff --git a/src/config.rs b/src/config.rs index 5ea636d8..b273c413 100644 --- a/src/config.rs +++ b/src/config.rs @@ -417,6 +417,8 @@ pub enum LlmBackend { Ollama, /// Any OpenAI-compatible endpoint (e.g. vLLM, LiteLLM, Together) OpenAiCompatible, + /// Tinfoil private inference + Tinfoil, } impl std::str::FromStr for LlmBackend { @@ -429,8 +431,9 @@ impl std::str::FromStr for LlmBackend { "anthropic" | "claude" => Ok(Self::Anthropic), "ollama" => Ok(Self::Ollama), "openai_compatible" | "openai-compatible" | "compatible" => Ok(Self::OpenAiCompatible), + "tinfoil" => Ok(Self::Tinfoil), _ => Err(format!( - "invalid LLM backend '{}', expected one of: nearai, openai, anthropic, ollama, openai_compatible", + "invalid LLM backend '{}', expected one of: nearai, openai, anthropic, ollama, openai_compatible, tinfoil", s )), } @@ -445,6 +448,7 @@ impl std::fmt::Display for LlmBackend { Self::Anthropic => write!(f, "anthropic"), Self::Ollama => write!(f, "ollama"), Self::OpenAiCompatible => write!(f, "openai_compatible"), + Self::Tinfoil => write!(f, "tinfoil"), } } } @@ -478,6 +482,13 @@ pub struct OpenAiCompatibleConfig { pub model: String, } +/// Configuration for Tinfoil private inference. +#[derive(Debug, Clone)] +pub struct TinfoilConfig { + pub api_key: SecretString, + pub model: String, +} + /// LLM provider configuration. /// /// NEAR AI remains the default backend. Users can switch to other providers @@ -496,6 +507,8 @@ pub struct LlmConfig { pub ollama: Option, /// OpenAI-compatible config (populated when backend=openai_compatible) pub openai_compatible: Option, + /// Tinfoil config (populated when backend=tinfoil) + pub tinfoil: Option, } /// API mode for NEAR AI. @@ -702,6 +715,19 @@ impl LlmConfig { None }; + let tinfoil = if backend == LlmBackend::Tinfoil { + let api_key = optional_env("TINFOIL_API_KEY")? + .map(SecretString::from) + .ok_or_else(|| ConfigError::MissingRequired { + key: "TINFOIL_API_KEY".to_string(), + hint: "Set TINFOIL_API_KEY when LLM_BACKEND=tinfoil".to_string(), + })?; + let model = optional_env("TINFOIL_MODEL")?.unwrap_or_else(|| "kimi-k2-5".to_string()); + Some(TinfoilConfig { api_key, model }) + } else { + None + }; + Ok(Self { backend, nearai, @@ -709,6 +735,7 @@ impl LlmConfig { anthropic, ollama, openai_compatible, + tinfoil, }) } } diff --git a/src/llm/mod.rs b/src/llm/mod.rs index 198d57c7..ef91f8e2 100644 --- a/src/llm/mod.rs +++ b/src/llm/mod.rs @@ -58,6 +58,7 @@ pub fn create_llm_provider( LlmBackend::Anthropic => create_anthropic_provider(config), LlmBackend::Ollama => create_ollama_provider(config), LlmBackend::OpenAiCompatible => create_openai_compatible_provider(config), + LlmBackend::Tinfoil => create_tinfoil_provider(config), } } @@ -154,6 +155,35 @@ fn create_ollama_provider(config: &LlmConfig) -> Result, Ll Ok(Arc::new(RigAdapter::new(model, &oll.model))) } +const TINFOIL_BASE_URL: &str = "https://inference.tinfoil.sh/v1"; + +fn create_tinfoil_provider(config: &LlmConfig) -> Result, LlmError> { + let tf = config + .tinfoil + .as_ref() + .ok_or_else(|| LlmError::AuthFailed { + provider: "tinfoil".to_string(), + })?; + + use rig::providers::openai; + + let client: openai::Client = openai::Client::builder() + .base_url(TINFOIL_BASE_URL) + .api_key(tf.api_key.expose_secret()) + .build() + .map_err(|e| LlmError::RequestFailed { + provider: "tinfoil".to_string(), + reason: format!("Failed to create Tinfoil client: {}", e), + })?; + + // Tinfoil currently only supports the Chat Completions API and not the newer Responses API, + // so we must explicitly select the completions API here (unlike other OpenAI-compatible providers). + let client = client.completions_api(); + let model = client.completion_model(&tf.model); + tracing::info!("Using Tinfoil private inference (model: {})", tf.model); + Ok(Arc::new(RigAdapter::new(model, &tf.model))) +} + fn create_openai_compatible_provider(config: &LlmConfig) -> Result, LlmError> { let compat = config .openai_compatible @@ -257,6 +287,7 @@ mod tests { anthropic: None, ollama: None, openai_compatible: None, + tinfoil: None, } } diff --git a/src/setup/wizard.rs b/src/setup/wizard.rs index 11a8d1e4..2625537e 100644 --- a/src/setup/wizard.rs +++ b/src/setup/wizard.rs @@ -1034,6 +1034,7 @@ impl SetupWizard { anthropic: None, ollama: None, openai_compatible: None, + tinfoil: None, }; match create_llm_provider(&config, session) {