From 96d5fc0d3956f5518b7c8845b92cf8a52ba54ec1 Mon Sep 17 00:00:00 2001 From: Jason Lee Date: Wed, 18 Feb 2026 13:59:34 +0800 Subject: [PATCH] feat: add Tinfoil private inference provider (#62) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add Tinfoil private inference provider Add a dedicated Tinfoil LLM backend (`LLM_BACKEND=tinfoil`) for Tinfoil's private inference service (https://tinfoil.sh). The existing `openai_compatible` backend cannot be used with Tinfoil because rig-core 0.30.0 defaults to the OpenAI Responses API (`/v1/responses`), which Tinfoil does not support — it only implements the Chat Completions API (`/v1/chat/completions`), returning 403 "shim: path not allowed" when hit on the responses endpoint. Rather than changing `openai_compatible` to use Chat Completions (which would break users expecting the Responses API), this adds a dedicated provider that explicitly uses rig's `.completions_api()` client. This also lays the groundwork for integrating Tinfoil's privacy wrapper client (enclave attestation, TLS certificate pinning) once their Rust SDK is available. The provider implementation can be swapped to use the Tinfoil Rust client without changing the LlmProvider interface. Configuration: LLM_BACKEND=tinfoil TINFOIL_API_KEY=tk_... TINFOIL_MODEL=kimi-k2-5 # optional, default * style: fix rustfmt formatting in Tinfoil provider * style: remove unnecessary tin_foil alias for Tinfoil backend * Update src/llm/mod.rs Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix: add tinfoil field to LlmConfig test fixture * style: fix rustfmt output in session manager --------- Co-authored-by: firat.sertgoz Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Illia Polosukhin --- src/config.rs | 29 ++++++++++++++++++++++++++++- src/llm/mod.rs | 31 +++++++++++++++++++++++++++++++ src/setup/wizard.rs | 1 + 3 files changed, 60 insertions(+), 1 deletion(-) diff --git a/src/config.rs b/src/config.rs index 5ea636d8..b273c413 100644 --- a/src/config.rs +++ b/src/config.rs @@ -417,6 +417,8 @@ pub enum LlmBackend { Ollama, /// Any OpenAI-compatible endpoint (e.g. vLLM, LiteLLM, Together) OpenAiCompatible, + /// Tinfoil private inference + Tinfoil, } impl std::str::FromStr for LlmBackend { @@ -429,8 +431,9 @@ impl std::str::FromStr for LlmBackend { "anthropic" | "claude" => Ok(Self::Anthropic), "ollama" => Ok(Self::Ollama), "openai_compatible" | "openai-compatible" | "compatible" => Ok(Self::OpenAiCompatible), + "tinfoil" => Ok(Self::Tinfoil), _ => Err(format!( - "invalid LLM backend '{}', expected one of: nearai, openai, anthropic, ollama, openai_compatible", + "invalid LLM backend '{}', expected one of: nearai, openai, anthropic, ollama, openai_compatible, tinfoil", s )), } @@ -445,6 +448,7 @@ impl std::fmt::Display for LlmBackend { Self::Anthropic => write!(f, "anthropic"), Self::Ollama => write!(f, "ollama"), Self::OpenAiCompatible => write!(f, "openai_compatible"), + Self::Tinfoil => write!(f, "tinfoil"), } } } @@ -478,6 +482,13 @@ pub struct OpenAiCompatibleConfig { pub model: String, } +/// Configuration for Tinfoil private inference. +#[derive(Debug, Clone)] +pub struct TinfoilConfig { + pub api_key: SecretString, + pub model: String, +} + /// LLM provider configuration. /// /// NEAR AI remains the default backend. Users can switch to other providers @@ -496,6 +507,8 @@ pub struct LlmConfig { pub ollama: Option, /// OpenAI-compatible config (populated when backend=openai_compatible) pub openai_compatible: Option, + /// Tinfoil config (populated when backend=tinfoil) + pub tinfoil: Option, } /// API mode for NEAR AI. @@ -702,6 +715,19 @@ impl LlmConfig { None }; + let tinfoil = if backend == LlmBackend::Tinfoil { + let api_key = optional_env("TINFOIL_API_KEY")? + .map(SecretString::from) + .ok_or_else(|| ConfigError::MissingRequired { + key: "TINFOIL_API_KEY".to_string(), + hint: "Set TINFOIL_API_KEY when LLM_BACKEND=tinfoil".to_string(), + })?; + let model = optional_env("TINFOIL_MODEL")?.unwrap_or_else(|| "kimi-k2-5".to_string()); + Some(TinfoilConfig { api_key, model }) + } else { + None + }; + Ok(Self { backend, nearai, @@ -709,6 +735,7 @@ impl LlmConfig { anthropic, ollama, openai_compatible, + tinfoil, }) } } diff --git a/src/llm/mod.rs b/src/llm/mod.rs index 198d57c7..ef91f8e2 100644 --- a/src/llm/mod.rs +++ b/src/llm/mod.rs @@ -58,6 +58,7 @@ pub fn create_llm_provider( LlmBackend::Anthropic => create_anthropic_provider(config), LlmBackend::Ollama => create_ollama_provider(config), LlmBackend::OpenAiCompatible => create_openai_compatible_provider(config), + LlmBackend::Tinfoil => create_tinfoil_provider(config), } } @@ -154,6 +155,35 @@ fn create_ollama_provider(config: &LlmConfig) -> Result, Ll Ok(Arc::new(RigAdapter::new(model, &oll.model))) } +const TINFOIL_BASE_URL: &str = "https://inference.tinfoil.sh/v1"; + +fn create_tinfoil_provider(config: &LlmConfig) -> Result, LlmError> { + let tf = config + .tinfoil + .as_ref() + .ok_or_else(|| LlmError::AuthFailed { + provider: "tinfoil".to_string(), + })?; + + use rig::providers::openai; + + let client: openai::Client = openai::Client::builder() + .base_url(TINFOIL_BASE_URL) + .api_key(tf.api_key.expose_secret()) + .build() + .map_err(|e| LlmError::RequestFailed { + provider: "tinfoil".to_string(), + reason: format!("Failed to create Tinfoil client: {}", e), + })?; + + // Tinfoil currently only supports the Chat Completions API and not the newer Responses API, + // so we must explicitly select the completions API here (unlike other OpenAI-compatible providers). + let client = client.completions_api(); + let model = client.completion_model(&tf.model); + tracing::info!("Using Tinfoil private inference (model: {})", tf.model); + Ok(Arc::new(RigAdapter::new(model, &tf.model))) +} + fn create_openai_compatible_provider(config: &LlmConfig) -> Result, LlmError> { let compat = config .openai_compatible @@ -257,6 +287,7 @@ mod tests { anthropic: None, ollama: None, openai_compatible: None, + tinfoil: None, } } diff --git a/src/setup/wizard.rs b/src/setup/wizard.rs index 11a8d1e4..2625537e 100644 --- a/src/setup/wizard.rs +++ b/src/setup/wizard.rs @@ -1034,6 +1034,7 @@ impl SetupWizard { anthropic: None, ollama: None, openai_compatible: None, + tinfoil: None, }; match create_llm_provider(&config, session) {