1176 lines · 43.5 KB · Rust
RawDownload
use anyhow::{Result, anyhow};use credentials_provider::CredentialsProvider;use fs::Fs;use futures::Stream;use futures::{FutureExt, StreamExt, future::BoxFuture, stream::BoxStream};use gpui::{App, AsyncApp, Context, Entity, Subscription, Task, TaskExt};use http_client::{CustomHeaders, HttpClient};use language_model::{    ApiKeyState, AuthenticateError, EnvVar, IconOrSvg, LanguageModel, LanguageModelCompletionError,    LanguageModelCompletionEvent, LanguageModelToolChoice, LanguageModelToolResultContent,    LanguageModelToolUse, MessageContent, StopReason, TokenUsage, env_var,};use language_model::{    InlineDescription, LanguageModelId, LanguageModelName, LanguageModelProvider,    LanguageModelProviderId, LanguageModelProviderName, LanguageModelProviderState,    LanguageModelRequest, ProviderSettingsView, RateLimiter, Role, SubPageProviderSettings,};use lmstudio::{LMSTUDIO_API_URL, ModelType, get_models}; pub use settings::LmStudioAvailableModel as AvailableModel;use settings::{Settings, SettingsStore, update_settings_file};use std::pin::Pin;use std::sync::LazyLock;use std::{    collections::{BTreeMap, HashMap},    sync::Arc,};use ui::{ButtonLike, ConfiguredApiCard, Divider, List, ListBulletItem, Tooltip, prelude::*};use ui_input::InputField; use crate::AllLanguageModelSettings;use language_model::util::parse_tool_arguments; const LMSTUDIO_DOWNLOAD_URL: &str = "https://lmstudio.ai/download";const LMSTUDIO_CATALOG_URL: &str = "https://lmstudio.ai/models";const LMSTUDIO_SITE: &str = "https://lmstudio.ai/"; const PROVIDER_ID: LanguageModelProviderId = LanguageModelProviderId::new("lmstudio");const PROVIDER_NAME: LanguageModelProviderName = LanguageModelProviderName::new("LM Studio"); const API_KEY_ENV_VAR_NAME: &str = "LMSTUDIO_API_KEY";static API_KEY_ENV_VAR: LazyLock<EnvVar> = env_var!(API_KEY_ENV_VAR_NAME); #[derive(Default, Debug, Clone, PartialEq)]pub struct LmStudioSettings {    pub api_url: String,    pub available_models: Vec<AvailableModel>,    pub custom_headers: CustomHeaders,} pub struct LmStudioLanguageModelProvider {    http_client: Arc<dyn HttpClient>,    state: Entity<State>,} pub struct State {    api_key_state: ApiKeyState,    credentials_provider: Arc<dyn CredentialsProvider>,    http_client: Arc<dyn HttpClient>,    available_models: Vec<lmstudio::Model>,    fetch_model_task: Option<Task<Result<()>>>,    _subscription: Subscription,} impl State {    fn is_authenticated(&self) -> bool {        !self.available_models.is_empty()    }     fn set_api_key(&mut self, api_key: Option<String>, cx: &mut Context<Self>) -> Task<Result<()>> {        let credentials_provider = self.credentials_provider.clone();        let api_url = LmStudioLanguageModelProvider::api_url(cx).into();        let task = self.api_key_state.store(            api_url,            api_key,            |this| &mut this.api_key_state,            credentials_provider,            cx,        );        self.restart_fetch_models_task(cx);        task    }     fn fetch_models(&mut self, cx: &mut Context<Self>) -> Task<Result<()>> {        let settings = &AllLanguageModelSettings::get_global(cx).lmstudio;        let http_client = self.http_client.clone();        let api_url = settings.api_url.clone();        let api_key = self.api_key_state.key(&api_url);        let extra_headers = settings.custom_headers.clone();         // As a proxy for the server being "authenticated", we'll check if its up by fetching the models        cx.spawn(async move |this, cx| {            let models = get_models(                http_client.as_ref(),                &api_url,                api_key.as_deref(),                None,                &extra_headers,            )            .await?;             let mut models: Vec<lmstudio::Model> = models                .into_iter()                .filter(|model| model.r#type != ModelType::Embeddings)                .map(|model| {                    lmstudio::Model::new(                        &model.id,                        None,                        model                            .loaded_context_length                            .or_else(|| model.max_context_length),                        model.capabilities.supports_tool_calls(),                        model.capabilities.supports_images() || model.r#type == ModelType::Vlm,                    )                })                .collect();             models.sort_by(|a, b| a.name.cmp(&b.name));             this.update(cx, |this, cx| {                this.available_models = models;                cx.notify();            })        })    }     fn restart_fetch_models_task(&mut self, cx: &mut Context<Self>) {        let task = self.fetch_models(cx);        self.fetch_model_task.replace(task);    }     fn authenticate(&mut self, cx: &mut Context<Self>) -> Task<Result<(), AuthenticateError>> {        let credentials_provider = self.credentials_provider.clone();        let api_url = LmStudioLanguageModelProvider::api_url(cx).into();        let _task = self.api_key_state.load_if_needed(            api_url,            |this| &mut this.api_key_state,            credentials_provider,            cx,        );         if self.is_authenticated() {            return Task::ready(Ok(()));        }         let fetch_models_task = self.fetch_models(cx);        cx.spawn(async move |_this, _cx| {            match fetch_models_task.await {                Ok(()) => Ok(()),                Err(err) => {                    // If any cause in the error chain is an std::io::Error with                    // ErrorKind::ConnectionRefused, treat this as "credentials not found"                    // (i.e. LM Studio not running).                    let mut connection_refused = false;                    for cause in err.chain() {                        if let Some(io_err) = cause.downcast_ref::<std::io::Error>() {                            if io_err.kind() == std::io::ErrorKind::ConnectionRefused {                                connection_refused = true;                                break;                            }                        }                    }                    if connection_refused {                        Err(AuthenticateError::ConnectionRefused)                    } else {                        Err(AuthenticateError::Other(err))                    }                }            }        })    }} impl LmStudioLanguageModelProvider {    pub fn new(        http_client: Arc<dyn HttpClient>,        credentials_provider: Arc<dyn CredentialsProvider>,        cx: &mut App,    ) -> Self {        let this = Self {            http_client: http_client.clone(),            state: cx.new(|cx| {                let subscription = cx.observe_global::<SettingsStore>({                    let mut settings = AllLanguageModelSettings::get_global(cx).lmstudio.clone();                    move |this: &mut State, cx| {                        let new_settings =                            AllLanguageModelSettings::get_global(cx).lmstudio.clone();                        if settings != new_settings {                            let credentials_provider = this.credentials_provider.clone();                            let api_url = Self::api_url(cx).into();                            this.api_key_state.handle_url_change(                                api_url,                                |this| &mut this.api_key_state,                                credentials_provider,                                cx,                            );                            settings = new_settings;                            this.restart_fetch_models_task(cx);                            cx.notify();                        }                    }                });                 State {                    api_key_state: ApiKeyState::new(                        Self::api_url(cx).into(),                        (*API_KEY_ENV_VAR).clone(),                    ),                    credentials_provider,                    http_client,                    available_models: Default::default(),                    fetch_model_task: None,                    _subscription: subscription,                }            }),        };        this.state            .update(cx, |state, cx| state.restart_fetch_models_task(cx));        this    }     fn api_url(cx: &App) -> String {        AllLanguageModelSettings::get_global(cx)            .lmstudio            .api_url            .clone()    }     fn has_custom_url(cx: &App) -> bool {        Self::api_url(cx) != LMSTUDIO_API_URL    }} impl LanguageModelProviderState for LmStudioLanguageModelProvider {    type ObservableEntity = State;     fn observable_entity(&self) -> Option<Entity<Self::ObservableEntity>> {        Some(self.state.clone())    }} impl LanguageModelProvider for LmStudioLanguageModelProvider {    fn id(&self) -> LanguageModelProviderId {        PROVIDER_ID    }     fn name(&self) -> LanguageModelProviderName {        PROVIDER_NAME    }     fn icon(&self) -> IconOrSvg {        IconOrSvg::Icon(IconName::AiLmStudio)    }     fn default_model(&self, _: &App) -> Option<Arc<dyn LanguageModel>> {        // We shouldn't try to select default model, because it might lead to a load call for an unloaded model.        // In a constrained environment where user might not have enough resources it'll be a bad UX to select something        // to load by default.        None    }     fn default_fast_model(&self, _: &App) -> Option<Arc<dyn LanguageModel>> {        // See explanation for default_model.        None    }     fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {        let mut models: BTreeMap<String, lmstudio::Model> = BTreeMap::default();         // Add models from the LM Studio API        for model in self.state.read(cx).available_models.iter() {            models.insert(model.name.clone(), model.clone());        }         // Override with available models from settings        for model in AllLanguageModelSettings::get_global(cx)            .lmstudio            .available_models            .iter()        {            models.insert(                model.name.clone(),                lmstudio::Model {                    name: model.name.clone(),                    display_name: model.display_name.clone(),                    max_tokens: model.max_tokens,                    supports_tool_calls: model.supports_tool_calls,                    supports_images: model.supports_images,                },            );        }         models            .into_values()            .map(|model| {                Arc::new(LmStudioLanguageModel {                    id: LanguageModelId::from(model.name.clone()),                    model,                    http_client: self.http_client.clone(),                    request_limiter: RateLimiter::new(4),                    state: self.state.clone(),                }) as Arc<dyn LanguageModel>            })            .collect()    }     fn is_authenticated(&self, cx: &App) -> bool {        self.state.read(cx).is_authenticated()    }     fn authenticate(&self, cx: &mut App) -> Task<Result<(), AuthenticateError>> {        self.state.update(cx, |state, cx| state.authenticate(cx))    }     fn settings_view(&self, _cx: &mut App) -> Option<ProviderSettingsView> {        let state = self.state.clone();        Some(ProviderSettingsView::SubPage(            SubPageProviderSettings::new(move |window, cx| {                cx.new(|cx| ConfigurationView::new(state.clone(), window, cx))                    .into()            })            .description(InlineDescription::Text(                "Run local LLMs like Llama, Phi, and Qwen with LM Studio.".into(),            )),        ))    }} pub struct LmStudioLanguageModel {    id: LanguageModelId,    model: lmstudio::Model,    http_client: Arc<dyn HttpClient>,    request_limiter: RateLimiter,    state: Entity<State>,} impl LmStudioLanguageModel {    fn to_lmstudio_request(        &self,        request: LanguageModelRequest,    ) -> Result<lmstudio::ChatCompletionRequest> {        if request.contains_custom_tool_input() {            anyhow::bail!("LM Studio does not support custom tools");        }         let mut messages = Vec::new();         for message in request.messages {            for content in message.content {                match content {                    MessageContent::Text(text) => add_message_content_part(                        lmstudio::MessagePart::Text { text },                        message.role,                        &mut messages,                    ),                    MessageContent::Thinking { .. } => {}                    MessageContent::RedactedThinking(_) => {}                    MessageContent::Compaction(_) => {}                    MessageContent::Image(image) => {                        add_message_content_part(                            lmstudio::MessagePart::Image {                                image_url: lmstudio::ImageUrl {                                    url: image.to_base64_url(),                                    detail: None,                                },                            },                            message.role,                            &mut messages,                        );                    }                    MessageContent::ToolUse(tool_use) => {                        let input = tool_use.input.as_json().ok_or_else(|| {                            anyhow!("LM Studio does not support custom tool calls")                        })?;                        let tool_call = lmstudio::ToolCall {                            id: tool_use.id.to_string(),                            content: lmstudio::ToolCallContent::Function {                                function: lmstudio::FunctionContent {                                    name: tool_use.name.to_string(),                                    arguments: serde_json::to_string(input).unwrap_or_default(),                                },                            },                        };                         if let Some(lmstudio::ChatMessage::Assistant { tool_calls, .. }) =                            messages.last_mut()                        {                            tool_calls.push(tool_call);                        } else {                            messages.push(lmstudio::ChatMessage::Assistant {                                content: None,                                tool_calls: vec![tool_call],                            });                        }                    }                    MessageContent::ToolResult(tool_result) => {                        let content: Vec<lmstudio::MessagePart> = tool_result                            .content                            .iter()                            .map(|part| match part {                                LanguageModelToolResultContent::Text(text) => {                                    lmstudio::MessagePart::Text {                                        text: text.to_string(),                                    }                                }                                LanguageModelToolResultContent::Image(image) => {                                    lmstudio::MessagePart::Image {                                        image_url: lmstudio::ImageUrl {                                            url: image.to_base64_url(),                                            detail: None,                                        },                                    }                                }                            })                            .collect();                         messages.push(lmstudio::ChatMessage::Tool {                            content: content.into(),                            tool_call_id: tool_result.tool_use_id.to_string(),                        });                    }                }            }        }         Ok(lmstudio::ChatCompletionRequest {            model: self.model.name.clone(),            messages,            stream: true,            stream_options: Some(lmstudio::StreamOptions {                include_usage: true,            }),            max_tokens: Some(-1),            stop: Some(request.stop),            // In LM Studio you can configure specific settings you'd like to use for your model.            // For example Qwen3 is recommended to be used with 0.7 temperature.            // It would be a bad UX to silently override these settings from Zed, so we pass no temperature as a default.            temperature: request.temperature.or(None),            tools: request                .tools                .into_iter()                .map(|tool| {                    let input_schema = match tool.input {                        language_model::LanguageModelRequestToolInput::Function {                            input_schema,                            ..                        } => input_schema,                        language_model::LanguageModelRequestToolInput::Custom { .. } => {                            return Err(anyhow::anyhow!("LM Studio does not support custom tools"));                        }                    };                    Ok(lmstudio::ToolDefinition::Function {                        function: lmstudio::FunctionDefinition {                            name: tool.name,                            description: Some(tool.description),                            parameters: Some(input_schema),                        },                    })                })                .collect::<Result<_>>()?,            tool_choice: request.tool_choice.map(|choice| match choice {                LanguageModelToolChoice::Auto => lmstudio::ToolChoice::Auto,                LanguageModelToolChoice::Any => lmstudio::ToolChoice::Required,                LanguageModelToolChoice::None => lmstudio::ToolChoice::None,            }),        })    }     fn stream_completion(        &self,        request: lmstudio::ChatCompletionRequest,        cx: &AsyncApp,    ) -> BoxFuture<        'static,        Result<futures::stream::BoxStream<'static, Result<lmstudio::ResponseStreamEvent>>>,    > {        let http_client = self.http_client.clone();        let (api_key, api_url, extra_headers) = self.state.read_with(cx, |state, cx| {            let api_url = LmStudioLanguageModelProvider::api_url(cx);            let extra_headers = AllLanguageModelSettings::get_global(cx)                .lmstudio                .custom_headers                .clone();            (state.api_key_state.key(&api_url), api_url, extra_headers)        });         let future = self.request_limiter.stream(async move {            let stream = lmstudio::stream_chat_completion(                http_client.as_ref(),                &api_url,                api_key.as_deref(),                request,                &extra_headers,            )            .await?;            Ok(stream)        });         async move { Ok(future.await?.boxed()) }.boxed()    }

Lines 1–500 of 1176.

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t