History

anthropic: Dynamically fetch models from `/models` (#56397)

Most compelling reason to make this change is that we don't have to ship a new Zed binary if Anthropic releases a new model Self-Review Checklist: - [x] I've reviewed my own diff for quality, security, and reliability - [x] Unsafe blocks (if any) have justifying comments - [x] The content is consistent with the [UI/UX checklist](https://github.com/zed-industries/zed/blob/main/CONTRIBUTING.md#uiux-checklist) - [x] Tests cover the new/changed behavior - [x] Performance impact has been considered and is acceptable Release Notes: - anthropic: Dynamically fetch available models from Anthropic API ---------

Bennet Bo Fenner committed 6 months ago

Commit ee309a01 parent 40d44443 files · +487 −500Browse files

Unified · Split

3 files
  1. crates/anthropic/src/anthropic.rs +288 −393
  2. crates/language_models/src/provider/anthropic.rs +199 −102
  3. docs/src/ai/llm-providers.md +0 −5
crates/anthropic/src/anthropic.rs+288−393
@@ -2,13 +2,13 @@ use std::io; · 1 unmodified line
2use std::str::FromStr;2use std::str::FromStr;
3use std::time::Duration;3use std::time::Duration;
4 4
5use anyhow::{Context as _, Result, anyhow};5use anyhow::{Context as _, Result};
6use chrono::{DateTime, Utc};6use chrono::{DateTime, Utc};
7use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream};7use futures::{AsyncBufReadExt, AsyncReadExt, StreamExt, io::BufReader, stream::BoxStream};
8use http_client::http::{self, HeaderMap, HeaderValue};8use http_client::http::{self, HeaderMap, HeaderValue};
9use http_client::{AsyncBody, HttpClient, Method, Request as HttpRequest, StatusCode};9use http_client::{AsyncBody, HttpClient, Method, Request as HttpRequest, StatusCode};
10use serde::{Deserialize, Serialize};10use serde::{Deserialize, Serialize};
11use strum::{EnumIter, EnumString};11use strum::EnumString;
12use thiserror::Error;12use thiserror::Error;
13 13
14pub mod batches;14pub mod batches;
@@ -16,14 +16,6 @@ pub mod completion; · 1 unmodified line
16 16
17pub const ANTHROPIC_API_URL: &str = "https://api.anthropic.com";17pub const ANTHROPIC_API_URL: &str = "https://api.anthropic.com";
18 18
19#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
20#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
21pub struct AnthropicModelCacheConfiguration {
22 pub min_total_token: u64,
23 pub should_speculate: bool,
24 pub max_cache_anchors: usize,
25}
26
27#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]19#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]
28#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]20#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq)]
29pub enum AnthropicModelMode {21pub enum AnthropicModelMode {
@@ -35,343 +27,152 @@ pub enum AnthropicModelMode { · 5 unmodified lines
35 AdaptiveThinking,27 AdaptiveThinking,
36}28}
37 29
38#[cfg_attr(feature = "schemars", derive(schemars::JsonSchema))]30/// Capabilities reported by the Anthropic models endpoint for a given model.
39#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, EnumIter)]31#[derive(Clone, Debug, Default, Deserialize)]
40pub enum Model {32pub struct ModelCapabilities {
41 #[serde(33 #[serde(default)]
42 rename = "claude-opus-4",34 pub thinking: Option<ThinkingCapability>,
43 alias = "claude-opus-4-latest",35 #[serde(default)]
44 alias = "claude-opus-4-thinking",36 pub image_input: Option<SupportedCapability>,
45 alias = "claude-opus-4-thinking-latest"37 #[serde(default)]
46 )]38 pub effort: Option<EffortCapability>,
47 ClaudeOpus4,
48 #[serde(
49 rename = "claude-opus-4-1",
50 alias = "claude-opus-4-1-latest",
51 alias = "claude-opus-4-1-thinking",
52 alias = "claude-opus-4-1-thinking-latest"
53 )]
54 ClaudeOpus4_1,
55 #[serde(
56 rename = "claude-opus-4-5",
57 alias = "claude-opus-4-5-latest",
58 alias = "claude-opus-4-5-thinking",
59 alias = "claude-opus-4-5-thinking-latest"
60 )]
61 ClaudeOpus4_5,
62 #[serde(
63 rename = "claude-opus-4-6",
64 alias = "claude-opus-4-6-latest",
65 alias = "claude-opus-4-6-1m-context",
66 alias = "claude-opus-4-6-1m-context-latest",
67 alias = "claude-opus-4-6-thinking",
68 alias = "claude-opus-4-6-thinking-latest",
69 alias = "claude-opus-4-6-1m-context-thinking",
70 alias = "claude-opus-4-6-1m-context-thinking-latest"
71 )]
72 ClaudeOpus4_6,
73 #[serde(
74 rename = "claude-opus-4-7",
75 alias = "claude-opus-4-7-latest",
76 alias = "claude-opus-4-7-1m-context",
77 alias = "claude-opus-4-7-1m-context-latest",
78 alias = "claude-opus-4-7-thinking",
79 alias = "claude-opus-4-7-thinking-latest",
80 alias = "claude-opus-4-7-1m-context-thinking",
81 alias = "claude-opus-4-7-1m-context-thinking-latest"
82 )]
83 ClaudeOpus4_7,
84 #[serde(
85 rename = "claude-sonnet-4",
86 alias = "claude-sonnet-4-latest",
87 alias = "claude-sonnet-4-thinking",
88 alias = "claude-sonnet-4-thinking-latest"
89 )]
90 ClaudeSonnet4,
91 #[serde(
92 rename = "claude-sonnet-4-5",
93 alias = "claude-sonnet-4-5-latest",
94 alias = "claude-sonnet-4-5-thinking",
95 alias = "claude-sonnet-4-5-thinking-latest"
96 )]
97 ClaudeSonnet4_5,
98 #[default]
99 #[serde(
100 rename = "claude-sonnet-4-6",
101 alias = "claude-sonnet-4-6-latest",
102 alias = "claude-sonnet-4-6-1m-context",
103 alias = "claude-sonnet-4-6-1m-context-latest",
104 alias = "claude-sonnet-4-6-thinking",
105 alias = "claude-sonnet-4-6-thinking-latest",
106 alias = "claude-sonnet-4-6-1m-context-thinking",
107 alias = "claude-sonnet-4-6-1m-context-thinking-latest"
108 )]
109 ClaudeSonnet4_6,
110 #[serde(
111 rename = "claude-haiku-4-5",
112 alias = "claude-haiku-4-5-latest",
113 alias = "claude-haiku-4-5-thinking",
114 alias = "claude-haiku-4-5-thinking-latest"
115 )]
116 ClaudeHaiku4_5,
117 #[serde(rename = "claude-3-haiku", alias = "claude-3-haiku-latest")]
118 Claude3Haiku,
119 #[serde(rename = "custom")]
120 Custom {
121 name: String,
122 max_tokens: u64,
123 /// The name displayed in the UI, such as in the agent panel model dropdown menu.
124 display_name: Option<String>,
125 /// Override this model with a different Anthropic model for tool calls.
126 tool_override: Option<String>,
127 /// Indicates whether this custom model supports caching.
128 cache_configuration: Option<AnthropicModelCacheConfiguration>,
129 max_output_tokens: Option<u64>,
130 default_temperature: Option<f32>,
131 #[serde(default)]
132 extra_beta_headers: Vec<String>,
133 #[serde(default)]
134 mode: AnthropicModelMode,
135 },
136}39}
137 40
138impl Model {41#[derive(Clone, Debug, Default, Deserialize)]
139 pub fn default_fast() -> Self {42pub struct SupportedCapability {
140 Self::ClaudeHaiku4_543 #[serde(default)]
141 }44 pub supported: bool,
142 45}
143 pub fn from_id(id: &str) -> Result<Self> {
144 if id.starts_with("claude-opus-4-7") {
145 return Ok(Self::ClaudeOpus4_7);
146 }
147
148 if id.starts_with("claude-opus-4-6") {
149 return Ok(Self::ClaudeOpus4_6);
150 }
151
152 if id.starts_with("claude-opus-4-5") {
153 return Ok(Self::ClaudeOpus4_5);
154 }
155
156 if id.starts_with("claude-opus-4-1") {
157 return Ok(Self::ClaudeOpus4_1);
158 }
159
160 if id.starts_with("claude-opus-4") {
161 return Ok(Self::ClaudeOpus4);
162 }
163
164 if id.starts_with("claude-sonnet-4-6") {
165 return Ok(Self::ClaudeSonnet4_6);
166 }
167
168 if id.starts_with("claude-sonnet-4-5") {
169 return Ok(Self::ClaudeSonnet4_5);
170 }
171
172 if id.starts_with("claude-sonnet-4") {
173 return Ok(Self::ClaudeSonnet4);
174 }
175
176 if id.starts_with("claude-haiku-4-5") {
177 return Ok(Self::ClaudeHaiku4_5);
178 }
179
180 if id.starts_with("claude-3-haiku") {
181 return Ok(Self::Claude3Haiku);
182 }
183
184 Err(anyhow!("invalid model ID: {id}"))
185 }
186
187 pub fn id(&self) -> &str {
188 match self {
189 Self::ClaudeOpus4 => "claude-opus-4-latest",
190 Self::ClaudeOpus4_1 => "claude-opus-4-1-latest",
191 Self::ClaudeOpus4_5 => "claude-opus-4-5-latest",
192 Self::ClaudeOpus4_6 => "claude-opus-4-6-latest",
193 Self::ClaudeOpus4_7 => "claude-opus-4-7-latest",
194 Self::ClaudeSonnet4 => "claude-sonnet-4-latest",
195 Self::ClaudeSonnet4_5 => "claude-sonnet-4-5-latest",
196 Self::ClaudeSonnet4_6 => "claude-sonnet-4-6-latest",
197 Self::ClaudeHaiku4_5 => "claude-haiku-4-5-latest",
198 Self::Claude3Haiku => "claude-3-haiku-20240307",
199 Self::Custom { name, .. } => name,
200 }
201 }
202
203 /// The id of the model that should be used for making API requests
204 pub fn request_id(&self) -> &str {
205 match self {
206 Self::ClaudeOpus4 => "claude-opus-4-20250514",
207 Self::ClaudeOpus4_1 => "claude-opus-4-1-20250805",
208 Self::ClaudeOpus4_5 => "claude-opus-4-5-20251101",
209 Self::ClaudeOpus4_6 => "claude-opus-4-6",
210 Self::ClaudeOpus4_7 => "claude-opus-4-7",
211 Self::ClaudeSonnet4 => "claude-sonnet-4-20250514",
212 Self::ClaudeSonnet4_5 => "claude-sonnet-4-5-20250929",
213 Self::ClaudeSonnet4_6 => "claude-sonnet-4-6",
214 Self::ClaudeHaiku4_5 => "claude-haiku-4-5-20251001",
215 Self::Claude3Haiku => "claude-3-haiku-20240307",
216 Self::Custom { name, .. } => name,
217 }
218 }
219 46
220 pub fn display_name(&self) -> &str {47#[derive(Clone, Debug, Default, Deserialize)]
221 match self {48pub struct ThinkingCapability {
222 Self::ClaudeOpus4 => "Claude Opus 4",49 #[serde(default)]
223 Self::ClaudeOpus4_1 => "Claude Opus 4.1",50 pub supported: bool,
224 Self::ClaudeOpus4_5 => "Claude Opus 4.5",51 #[serde(default)]
225 Self::ClaudeOpus4_6 => "Claude Opus 4.6",52 pub types: Option<ThinkingTypes>,
226 Self::ClaudeOpus4_7 => "Claude Opus 4.7",53}
227 Self::ClaudeSonnet4 => "Claude Sonnet 4",
228 Self::ClaudeSonnet4_5 => "Claude Sonnet 4.5",
229 Self::ClaudeSonnet4_6 => "Claude Sonnet 4.6",
230 Self::ClaudeHaiku4_5 => "Claude Haiku 4.5",
231 Self::Claude3Haiku => "Claude 3 Haiku",
232 Self::Custom {
233 name, display_name, ..
234 } => display_name.as_ref().unwrap_or(name),
235 }
236 }
237 54
238 pub fn cache_configuration(&self) -> Option<AnthropicModelCacheConfiguration> {55#[derive(Clone, Debug, Default, Deserialize)]
239 match self {56pub struct ThinkingTypes {
240 Self::ClaudeOpus457 #[serde(default)]
241 | Self::ClaudeOpus4_158 pub adaptive: SupportedCapability,
242 | Self::ClaudeOpus4_559 #[serde(default)]
243 | Self::ClaudeOpus4_660 pub enabled: SupportedCapability,
244 | Self::ClaudeOpus4_761}
245 | Self::ClaudeSonnet4
246 | Self::ClaudeSonnet4_5
247 | Self::ClaudeSonnet4_6
248 | Self::ClaudeHaiku4_5
249 | Self::Claude3Haiku => Some(AnthropicModelCacheConfiguration {
250 min_total_token: 2_048,
251 should_speculate: true,
252 max_cache_anchors: 4,
253 }),
254 Self::Custom {
255 cache_configuration,
256 ..
257 } => cache_configuration.clone(),
258 }
259 }
260 62
261 pub fn max_token_count(&self) -> u64 {63#[derive(Clone, Debug, Default, Deserialize)]
262 match self {64pub struct EffortCapability {
263 Self::ClaudeOpus465 #[serde(default)]
264 | Self::ClaudeOpus4_166 pub supported: bool,
265 | Self::ClaudeOpus4_567 #[serde(default)]
266 | Self::ClaudeSonnet468 pub low: Option<SupportedCapability>,
267 | Self::ClaudeSonnet4_569 #[serde(default)]
268 | Self::ClaudeHaiku4_570 pub medium: Option<SupportedCapability>,
269 | Self::Claude3Haiku => 200_000,71 #[serde(default)]
270 Self::ClaudeOpus4_6 | Self::ClaudeOpus4_7 | Self::ClaudeSonnet4_6 => 1_000_000,72 pub high: Option<SupportedCapability>,
271 Self::Custom { max_tokens, .. } => *max_tokens,73 #[serde(default)]
272 }74 pub max: Option<SupportedCapability>,
273 }75 #[serde(default)]
76 pub xhigh: Option<SupportedCapability>,
77}
274 78
275 pub fn max_output_tokens(&self) -> u64 {79#[derive(Clone, Debug, PartialEq)]
276 match self {80pub struct Model {
277 Self::ClaudeOpus4 | Self::ClaudeOpus4_1 => 32_000,81 pub id: String,
278 Self::ClaudeOpus4_582 pub display_name: String,
279 | Self::ClaudeSonnet483 pub max_input_tokens: u64,
280 | Self::ClaudeSonnet4_584 pub max_output_tokens: u64,
281 | Self::ClaudeSonnet4_685 pub default_temperature: f32,
282 | Self::ClaudeHaiku4_5 => 64_000,86 pub mode: AnthropicModelMode,
283 Self::ClaudeOpus4_6 | Self::ClaudeOpus4_7 => 128_000,87 pub supports_thinking: bool,
284 Self::Claude3Haiku => 4_096,88 pub supports_adaptive_thinking: bool,
285 Self::Custom {89 pub supports_images: bool,
286 max_output_tokens, ..90 pub supports_speed: bool,
287 } => max_output_tokens.unwrap_or(4_096),91 pub supported_effort_levels: Vec<Effort>,
288 }92 /// A model id to substitute when invoking tools, used for models that
289 }93 /// don't support tool calling natively.
94 pub tool_override: Option<String>,
95 /// Extra `Anthropic-Beta` header values to send with each request.
96 pub extra_beta_headers: Vec<String>,
97}
290 98
291 pub fn default_temperature(&self) -> f32 {99impl Model {
292 match self {100 /// Construct a `Model` from an entry returned by the `/v1/models` listing endpoint.
293 Self::ClaudeOpus4101 pub fn from_listed(entry: ListModelEntry) -> Self {
294 | Self::ClaudeOpus4_1102 let supports_thinking = entry
295 | Self::ClaudeOpus4_5103 .capabilities
296 | Self::ClaudeOpus4_6104 .as_ref()
297 | Self::ClaudeOpus4_7105 .and_then(|t| t.thinking.as_ref())
298 | Self::ClaudeSonnet4106 .map(|t| t.supported)
299 | Self::ClaudeSonnet4_5107 .unwrap_or(false);
300 | Self::ClaudeSonnet4_6108 let supports_adaptive_thinking = entry
301 | Self::ClaudeHaiku4_5109 .capabilities
302 | Self::Claude3Haiku => 1.0,110 .as_ref()
303 Self::Custom {111 .and_then(|t| t.thinking.as_ref())
304 default_temperature,112 .and_then(|t| t.types.as_ref())
305 ..113 .map(|types| types.adaptive.supported)
306 } => default_temperature.unwrap_or(1.0),114 .unwrap_or(false);
115 let supports_images = entry
116 .capabilities
117 .as_ref()
118 .and_then(|c| c.image_input.as_ref())
119 .map(|c| c.supported)
120 .unwrap_or(false);
121
122 let mut supported_effort_levels = Vec::new();
123 if let Some(effort) = entry.capabilities.as_ref().and_then(|e| e.effort.as_ref()) {
124 // The `xhigh` effort level reported by the API has no
125 // corresponding `Effort` variant in the request enum, so it is
126 // intentionally dropped here.
127 for (level, supported) in [
128 (Effort::Low, effort.low.as_ref()),
129 (Effort::Medium, effort.medium.as_ref()),
130 (Effort::High, effort.high.as_ref()),
131 (Effort::XHigh, effort.xhigh.as_ref()),
132 (Effort::Max, effort.max.as_ref()),
133 ] {
134 if supported.map(|c| c.supported).unwrap_or(false) {
135 supported_effort_levels.push(level);
136 }
137 }
307 }138 }
308 }
309 139
310 pub fn mode(&self) -> AnthropicModelMode {140 let mode = if supports_adaptive_thinking {
311 match self {141 AnthropicModelMode::AdaptiveThinking
312 Self::Custom { mode, .. } => mode.clone(),142 } else if supports_thinking {
313 _ if self.supports_adaptive_thinking() => AnthropicModelMode::AdaptiveThinking,143 AnthropicModelMode::Thinking {
314 _ if self.supports_thinking() => AnthropicModelMode::Thinking {
315 budget_tokens: Some(4_096),144 budget_tokens: Some(4_096),
316 },
317 _ => AnthropicModelMode::Default,
318 }
319 }
320
321 pub fn supports_thinking(&self) -> bool {
322 match self {
323 Self::Custom { mode, .. } => {
324 matches!(
325 mode,
326 AnthropicModelMode::Thinking { .. } | AnthropicModelMode::AdaptiveThinking
327 )
328 }145 }
329 _ => matches!(146 } else {
330 self,147 AnthropicModelMode::Default
331 Self::ClaudeOpus4148 };
332 | Self::ClaudeOpus4_1
333 | Self::ClaudeOpus4_5
334 | Self::ClaudeOpus4_6
335 | Self::ClaudeOpus4_7
336 | Self::ClaudeSonnet4
337 | Self::ClaudeSonnet4_5
338 | Self::ClaudeSonnet4_6
339 | Self::ClaudeHaiku4_5
340 ),
341 }
342 }
343 149
344 pub fn supports_speed(&self) -> bool {150 let supports_speed = entry.id == "claude-opus-4-6";
345 matches!(self, Self::ClaudeOpus4_6 | Self::ClaudeSonnet4_6)
346 }
347 151
348 pub fn supports_adaptive_thinking(&self) -> bool {152 Self {
349 match self {153 display_name: entry.display_name,
350 Self::Custom { mode, .. } => matches!(mode, AnthropicModelMode::AdaptiveThinking),154 id: entry.id,
351 _ => matches!(155 max_input_tokens: entry.max_input_tokens,
352 self,156 max_output_tokens: entry.max_tokens,
353 Self::ClaudeOpus4_6 | Self::ClaudeOpus4_7 | Self::ClaudeSonnet4_6157 default_temperature: 1.0,
354 ),158 mode,
159 supports_thinking,
160 supports_adaptive_thinking,
161 supports_images,
162 supports_speed,
163 supported_effort_levels,
164 tool_override: None,
165 extra_beta_headers: Vec::new(),
355 }166 }
356 }167 }
357 168
358 pub fn beta_headers(&self) -> Option<String> {169 pub fn beta_headers(&self) -> Option<String> {
359 let mut headers = vec![];170 let headers: Vec<&str> = self
360 171 .extra_beta_headers
361 match self {172 .iter()
362 Self::Custom {173 .map(|h| h.trim())
363 extra_beta_headers, ..174 .filter(|h| !h.is_empty())
364 } => {175 .collect();
365 headers.extend(
366 extra_beta_headers
367 .iter()
368 .filter(|header| !header.trim().is_empty())
369 .cloned(),
370 );
371 }
372 _ => {}
373 }
374
375 if headers.is_empty() {176 if headers.is_empty() {
376 None177 None
377 } else {178 } else {
@@ -379,15 +180,11 @@ impl Model { · 1 unmodified line
379 }180 }
380 }181 }
381 182
382 pub fn tool_model_id(&self) -> &str {183 pub fn request_id(&self, has_tools: bool) -> &str {
383 if let Self::Custom {184 if has_tools {
384 tool_override: Some(tool_override),185 self.tool_override.as_deref().unwrap_or(&self.id)
385 ..
386 } = self
387 {
388 tool_override
389 } else {186 } else {
390 self.request_id()187 &self.id
391 }188 }
392 }189 }
393}190}
@@ -405,6 +202,73 @@ pub async fn stream_completion( · 11 unmodified lines
405 .map(|output| output.0)202 .map(|output| output.0)
406}203}
407 204
205/// A raw model entry returned by the Anthropic models listing endpoint.
206#[derive(Clone, Debug, Deserialize)]
207pub struct ListModelEntry {
208 pub id: String,
209 pub display_name: String,
210 pub max_input_tokens: u64,
211 pub max_tokens: u64,
212 #[serde(default)]
213 pub capabilities: Option<ModelCapabilities>,
214}
215
216#[derive(Debug, Deserialize)]
217struct ListModelsResponse {
218 data: Vec<ListModelEntry>,
219}
220
221/// Fetch the list of models available to the current API key. The returned
222/// models are constructed by feeding each raw entry through
223/// [`Model::from_listed`].
224///
225/// See https://docs.claude.com/en/api/models-list.
226pub async fn list_models(
227 client: &dyn HttpClient,
228 api_url: &str,
229 api_key: &str,
230) -> Result<Vec<Model>> {
231 let uri = format!("{api_url}/v1/models?limit=1000");
232
233 let request = HttpRequest::builder()
234 .method(Method::GET)
235 .uri(uri)
236 .header("Anthropic-Version", "2023-06-01")
237 .header("X-Api-Key", api_key.trim())
238 .header("Accept", "application/json")
239 .body(AsyncBody::default())
240 .context("failed to build Anthropic models list request")?;
241
242 let mut response = client
243 .send(request)
244 .await
245 .context("failed to send Anthropic models list request")?;
246
247 let mut body = String::new();
248 response
249 .body_mut()
250 .read_to_string(&mut body)
251 .await
252 .context("failed to read Anthropic models list response")?;
253
254 anyhow::ensure!(
255 response.status().is_success(),
256 "failed to list Anthropic models: {} {}",
257 response.status(),
258 body,
259 );
260
261 let parsed: ListModelsResponse =
262 serde_json::from_str(&body).context("failed to parse Anthropic models list response")?;
263
264 let models = parsed
265 .data
266 .into_iter()
267 .map(Model::from_listed)
268 .collect::<Vec<_>>();
269 Ok(models)
270}
271
408/// Generate completion without streaming.272/// Generate completion without streaming.
409pub async fn non_streaming_completion(273pub async fn non_streaming_completion(
410 client: &dyn HttpClient,274 client: &dyn HttpClient,
@@ -780,13 +644,14 @@ pub enum AdaptiveThinkingDisplay { · 369 unmodified lines
780 Summarized,644 Summarized,
781}645}
782 646
783#[derive(Debug, Clone, Copy, Serialize, Deserialize, EnumString)]647#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, EnumString)]
784#[serde(rename_all = "snake_case")]648#[serde(rename_all = "snake_case")]
785#[strum(serialize_all = "snake_case")]649#[strum(serialize_all = "snake_case")]
786pub enum Effort {650pub enum Effort {
787 Low,651 Low,
788 Medium,652 Medium,
789 High,653 High,
654 XHigh,
790 Max,655 Max,
791}656}
792 657
@@ -1095,63 +960,93 @@ impl From<ApiError> for language_model_core::LanguageModelCompletionError { · 302 unmodified lines
1095 }960 }
1096}961}
1097 962
1098#[test]963#[cfg(test)]
1099fn custom_mode_thinking_is_preserved() {964mod tests {
1100 let model = Model::Custom {965 use super::*;
1101 name: "my-custom-model".to_string(),966
1102 max_tokens: 8192,967 fn listed_entry(id: &str, capabilities: ModelCapabilities) -> ListModelEntry {
1103 display_name: None,968 ListModelEntry {
1104 tool_override: None,969 id: id.to_string(),
1105 cache_configuration: None,970 display_name: id.to_string(),
1106 max_output_tokens: None,971 max_input_tokens: 200_000,
1107 default_temperature: None,972 max_tokens: 64_000,
1108 extra_beta_headers: vec![],973 capabilities: Some(capabilities),
1109 mode: AnthropicModelMode::Thinking {
1110 budget_tokens: Some(2048),
1111 },
1112 };
1113 assert_eq!(
1114 model.mode(),
1115 AnthropicModelMode::Thinking {
1116 budget_tokens: Some(2048)
1117 }974 }
1118 );975 }
1119 assert!(model.supports_thinking());
1120}
1121 976
1122#[test]977 #[test]
1123fn custom_mode_adaptive_is_preserved() {978 fn from_listed_picks_adaptive_thinking_mode() {
1124 let model = Model::Custom {979 let entry = listed_entry(
1125 name: "my-custom-model".to_string(),980 "claude-test-adaptive",
1126 max_tokens: 8192,981 ModelCapabilities {
1127 display_name: None,982 thinking: Some(ThinkingCapability {
1128 tool_override: None,983 supported: true,
1129 cache_configuration: None,984 types: Some(ThinkingTypes {
1130 max_output_tokens: None,985 adaptive: SupportedCapability { supported: true },
1131 default_temperature: None,986 enabled: SupportedCapability { supported: true },
1132 extra_beta_headers: vec![],987 }),
1133 mode: AnthropicModelMode::AdaptiveThinking,988 }),
1134 };989 ..Default::default()
1135 assert_eq!(model.mode(), AnthropicModelMode::AdaptiveThinking);990 },
1136 assert!(model.supports_adaptive_thinking());991 );
1137 assert!(model.supports_thinking());992 let model = Model::from_listed(entry);
1138}993 assert!(model.supports_thinking);
994 assert!(model.supports_adaptive_thinking);
995 assert_eq!(model.mode, AnthropicModelMode::AdaptiveThinking);
996 }
1139 997
1140#[test]998 #[test]
1141fn custom_mode_default_disables_thinking() {999 fn from_listed_picks_thinking_mode_when_only_enabled_supported() {
1142 let model = Model::Custom {1000 let entry = listed_entry(
1143 name: "my-custom-model".to_string(),1001 "claude-test-thinking",
1144 max_tokens: 8192,1002 ModelCapabilities {
1145 display_name: None,1003 thinking: Some(ThinkingCapability {
1146 tool_override: None,1004 supported: true,
1147 cache_configuration: None,1005 types: Some(ThinkingTypes {
1148 max_output_tokens: None,1006 adaptive: SupportedCapability { supported: false },
1149 default_temperature: None,1007 enabled: SupportedCapability { supported: true },
1150 extra_beta_headers: vec![],1008 }),
1151 mode: AnthropicModelMode::Default,1009 }),
1152 };1010 ..Default::default()
1153 assert!(!model.supports_thinking());1011 },
1154 assert!(!model.supports_adaptive_thinking());1012 );
1013 let model = Model::from_listed(entry);
1014 assert!(model.supports_thinking);
1015 assert!(!model.supports_adaptive_thinking);
1016 assert!(matches!(model.mode, AnthropicModelMode::Thinking { .. }));
1017 }
1018
1019 #[test]
1020 fn from_listed_default_mode_when_no_thinking() {
1021 let entry = listed_entry("claude-test-default", ModelCapabilities::default());
1022 let model = Model::from_listed(entry);
1023 assert!(!model.supports_thinking);
1024 assert!(!model.supports_adaptive_thinking);
1025 assert_eq!(model.mode, AnthropicModelMode::Default);
1026 }
1027
1028 #[test]
1029 fn from_listed_collects_supported_effort_levels() {
1030 let entry = listed_entry(
1031 "claude-test-effort",
1032 ModelCapabilities {
1033 effort: Some(EffortCapability {
1034 supported: true,
1035 low: Some(SupportedCapability { supported: true }),
1036 medium: Some(SupportedCapability { supported: false }),
1037 high: Some(SupportedCapability { supported: true }),
1038 max: Some(SupportedCapability { supported: true }),
1039 xhigh: Some(SupportedCapability { supported: true }),
1040 }),
1041 ..Default::default()
1042 },
1043 );
1044 let model = Model::from_listed(entry);
1045 assert_eq!(
1046 &model.supported_effort_levels,
1047 &[Effort::Low, Effort::High, Effort::XHigh, Effort::Max]
1048 );
1049 }
1155}1050}
1156 1051
1157#[test]1052#[test]
crates/language_models/src/provider/anthropic.rs+199−102
@@ -17,7 +17,6 @@ use language_model::{ · 16 unmodified lines
17};17};
18use settings::{Settings, SettingsStore};18use settings::{Settings, SettingsStore};
19use std::sync::{Arc, LazyLock};19use std::sync::{Arc, LazyLock};
20use strum::IntoEnumIterator;
21use ui::{ButtonLink, ConfiguredApiCard, List, ListBulletItem, prelude::*};20use ui::{ButtonLink, ConfiguredApiCard, List, ListBulletItem, prelude::*};
22use ui_input::InputField;21use ui_input::InputField;
23use util::ResultExt;22use util::ResultExt;
@@ -46,6 +45,9 @@ static API_KEY_ENV_VAR: LazyLock<EnvVar> = env_var!(API_KEY_ENV_VAR_NAME); · 22 unmodified lines
46pub struct State {45pub struct State {
47 api_key_state: ApiKeyState,46 api_key_state: ApiKeyState,
48 credentials_provider: Arc<dyn CredentialsProvider>,47 credentials_provider: Arc<dyn CredentialsProvider>,
48 http_client: Arc<dyn HttpClient>,
49 fetched_models: Vec<anthropic::Model>,
50 fetch_models_task: Option<Task<Result<()>>>,
49}51}
50 52
51impl State {53impl State {
@@ -56,24 +58,68 @@ impl State { · 4 unmodified lines
56 fn set_api_key(&mut self, api_key: Option<String>, cx: &mut Context<Self>) -> Task<Result<()>> {58 fn set_api_key(&mut self, api_key: Option<String>, cx: &mut Context<Self>) -> Task<Result<()>> {
57 let credentials_provider = self.credentials_provider.clone();59 let credentials_provider = self.credentials_provider.clone();
58 let api_url = AnthropicLanguageModelProvider::api_url(cx);60 let api_url = AnthropicLanguageModelProvider::api_url(cx);
59 self.api_key_state.store(61 let should_fetch_models = api_key.is_some();
62 let task = self.api_key_state.store(
60 api_url,63 api_url,
61 api_key,64 api_key,
62 |this| &mut this.api_key_state,65 |this| &mut this.api_key_state,
63 credentials_provider,66 credentials_provider,
64 cx,67 cx,
65 )68 );
69 self.fetched_models.clear();
70 cx.spawn(async move |this, cx| {
71 let result = task.await;
72 if result.is_ok() && should_fetch_models {
73 this.update(cx, |this, cx| this.restart_fetch_models_task(cx))
74 .ok();
75 }
76 result
77 })
66 }78 }
67 79
68 fn authenticate(&mut self, cx: &mut Context<Self>) -> Task<Result<(), AuthenticateError>> {80 fn authenticate(&mut self, cx: &mut Context<Self>) -> Task<Result<(), AuthenticateError>> {
69 let credentials_provider = self.credentials_provider.clone();81 let credentials_provider = self.credentials_provider.clone();
70 let api_url = AnthropicLanguageModelProvider::api_url(cx);82 let api_url = AnthropicLanguageModelProvider::api_url(cx);
71 self.api_key_state.load_if_needed(83 let task = self.api_key_state.load_if_needed(
72 api_url,84 api_url,
73 |this| &mut this.api_key_state,85 |this| &mut this.api_key_state,
74 credentials_provider,86 credentials_provider,
75 cx,87 cx,
76 )88 );
89
90 cx.spawn(async move |this, cx| {
91 let result = task.await;
92 if result.is_ok() {
93 this.update(cx, |this, cx| this.restart_fetch_models_task(cx))
94 .ok();
95 }
96 result
97 })
98 }
99
100 fn fetch_models(&mut self, cx: &mut Context<Self>) -> Task<Result<()>> {
101 let http_client = self.http_client.clone();
102 let api_url = AnthropicLanguageModelProvider::api_url(cx);
103 let Some(api_key) = self.api_key_state.key(&api_url) else {
104 return Task::ready(Err(anyhow::anyhow!(
105 "cannot fetch Anthropic models without an API key"
106 )));
107 };
108
109 cx.spawn(async move |this, cx| {
110 let models =
111 anthropic::list_models(http_client.as_ref(), &api_url, api_key.as_ref()).await?;
112
113 this.update(cx, |this, cx| {
114 this.fetched_models = models;
115 cx.notify();
116 })
117 })
118 }
119
120 fn restart_fetch_models_task(&mut self, cx: &mut Context<Self>) {
121 let task = self.fetch_models(cx);
122 self.fetch_models_task.replace(task);
77 }123 }
78}124}
79 125
@@ -84,21 +130,33 @@ impl AnthropicLanguageModelProvider { · 4 unmodified lines
84 cx: &mut App,130 cx: &mut App,
85 ) -> Self {131 ) -> Self {
86 let state = cx.new(|cx| {132 let state = cx.new(|cx| {
87 cx.observe_global::<SettingsStore>(|this: &mut State, cx| {133 cx.observe_global::<SettingsStore>({
88 let credentials_provider = this.credentials_provider.clone();134 let mut last_api_url = Self::api_url(cx);
89 let api_url = Self::api_url(cx);135 move |this: &mut State, cx| {
90 this.api_key_state.handle_url_change(136 let credentials_provider = this.credentials_provider.clone();
91 api_url,137 let api_url = Self::api_url(cx);
92 |this| &mut this.api_key_state,138 let url_changed = api_url != last_api_url;
93 credentials_provider,139 last_api_url = api_url.clone();
94 cx,140 this.api_key_state.handle_url_change(
95 );141 api_url,
96 cx.notify();142 |this| &mut this.api_key_state,
143 credentials_provider,
144 cx,
145 );
146 if url_changed {
147 this.fetched_models.clear();
148 this.authenticate(cx).detach();
149 }
150 cx.notify();
151 }
97 })152 })
98 .detach();153 .detach();
99 State {154 State {
100 api_key_state: ApiKeyState::new(Self::api_url(cx), (*API_KEY_ENV_VAR).clone()),155 api_key_state: ApiKeyState::new(Self::api_url(cx), (*API_KEY_ENV_VAR).clone()),
101 credentials_provider,156 credentials_provider,
157 http_client: http_client.clone(),
158 fetched_models: Vec::new(),
159 fetch_models_task: None,
102 }160 }
103 });161 });
104 162
@@ -107,7 +165,7 @@ impl AnthropicLanguageModelProvider { · 2 unmodified lines
107 165
108 fn create_language_model(&self, model: anthropic::Model) -> Arc<dyn LanguageModel> {166 fn create_language_model(&self, model: anthropic::Model) -> Arc<dyn LanguageModel> {
109 Arc::new(AnthropicModel {167 Arc::new(AnthropicModel {
110 id: LanguageModelId::from(model.id().to_string()),168 id: LanguageModelId::from(model.id.to_string()),
111 model,169 model,
112 state: self.state.clone(),170 state: self.state.clone(),
113 http_client: self.http_client.clone(),171 http_client: self.http_client.clone(),
@@ -150,58 +208,44 @@ impl LanguageModelProvider for AnthropicLanguageModelProvider { · 36 unmodified lines
150 IconOrSvg::Icon(IconName::AiAnthropic)208 IconOrSvg::Icon(IconName::AiAnthropic)
151 }209 }
152 210
153 fn default_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {211 fn default_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
154 Some(self.create_language_model(anthropic::Model::default()))212 let fetched = self.state.read(cx).fetched_models.clone();
213 // Pick the highest-version Sonnet we know about; otherwise the first
214 // Claude model returned. Returning `None` until the fetch completes
215 // matches the Ollama provider's behavior.
216 pick_preferred_model(&fetched, &["claude-sonnet-", "claude-opus-", "claude-"])
217 .map(|model| self.create_language_model(model))
155 }218 }
156 219
157 fn default_fast_model(&self, _cx: &App) -> Option<Arc<dyn LanguageModel>> {220 fn default_fast_model(&self, cx: &App) -> Option<Arc<dyn LanguageModel>> {
158 Some(self.create_language_model(anthropic::Model::default_fast()))221 let fetched = self.state.read(cx).fetched_models.clone();
222 pick_preferred_model(&fetched, &["claude-haiku-", "claude-"])
223 .map(|model| self.create_language_model(model))
159 }224 }
160 225
161 fn recommended_models(&self, _cx: &App) -> Vec<Arc<dyn LanguageModel>> {226 fn recommended_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
162 [anthropic::Model::ClaudeSonnet4_6]227 let fetched = self.state.read(cx).fetched_models.clone();
163 .into_iter()228 pick_preferred_model(&fetched, &["claude-sonnet-"])
164 .map(|model| self.create_language_model(model))229 .map(|model| vec![self.create_language_model(model)])
165 .collect()230 .unwrap_or_default()
166 }231 }
167 232
168 fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {233 fn provided_models(&self, cx: &App) -> Vec<Arc<dyn LanguageModel>> {
169 let mut models = BTreeMap::default();234 let mut models: BTreeMap<String, anthropic::Model> = BTreeMap::default();
170 235
171 // Add base models from anthropic::Model::iter()236 // Models reported by Anthropic's `/v1/models` endpoint are the
172 for model in anthropic::Model::iter() {237 // primary source. The list will be empty until authentication has
173 if !matches!(model, anthropic::Model::Custom { .. }) {238 // succeeded and the first fetch completes.
174 models.insert(model.id().to_string(), model);239 for model in &self.state.read(cx).fetched_models {
175 }240 models.insert(model.id.to_string(), model.clone());
176 }241 }
177 242
178 // Override with available models from settings243 // User-defined `available_models` from settings can either add
179 for model in &AnthropicLanguageModelProvider::settings(cx).available_models {244 // entirely new entries or override fields on a fetched model with
180 models.insert(245 // the same id (e.g. enable Fast mode or set a tool override).
181 model.name.clone(),246 for available in &AnthropicLanguageModelProvider::settings(cx).available_models {
182 anthropic::Model::Custom {247 let model = available_model_to_anthropic_model(available);
183 name: model.name.clone(),248 models.insert(model.id.to_string(), model);
184 display_name: model.display_name.clone(),
185 max_tokens: model.max_tokens,
186 tool_override: model.tool_override.clone(),
187 cache_configuration: model.cache_configuration.as_ref().map(|config| {
188 anthropic::AnthropicModelCacheConfiguration {
189 max_cache_anchors: config.max_cache_anchors,
190 should_speculate: config.should_speculate,
191 min_total_token: config.min_total_token,
192 }
193 }),
194 max_output_tokens: model.max_output_tokens,
195 default_temperature: model.default_temperature,
196 extra_beta_headers: model.extra_beta_headers.clone(),
197 mode: match model.mode.unwrap_or_default() {
198 settings::ModelMode::Default => AnthropicModelMode::Default,
199 settings::ModelMode::Thinking { budget_tokens } => {
200 AnthropicModelMode::Thinking { budget_tokens }
201 }
202 },
203 },
204 );
205 }249 }
206 250
207 models251 models
@@ -234,6 +278,70 @@ impl LanguageModelProvider for AnthropicLanguageModelProvider { · 26 unmodified lines
234 }278 }
235}279}
236 280
281/// Pick the model from `models` whose id starts with the earliest matching
282/// prefix in `preferred_prefixes`. Within a single prefix bucket the model
283/// with the lexicographically greatest id wins, which roughly corresponds to
284/// the highest version since Anthropic ids embed dated suffixes.
285fn pick_preferred_model(
286 models: &[anthropic::Model],
287 preferred_prefixes: &[&str],
288) -> Option<anthropic::Model> {
289 for prefix in preferred_prefixes {
290 let candidate = models
291 .iter()
292 .filter(|m| m.id.starts_with(prefix))
293 .max_by(|a, b| a.id.cmp(&b.id));
294 if let Some(model) = candidate {
295 return Some(model.clone());
296 }
297 }
298 None
299}
300
301/// Convert a settings-defined `available_models` entry into an `anthropic::Model`.
302fn available_model_to_anthropic_model(available: &AvailableModel) -> anthropic::Model {
303 let mode = match available.mode.unwrap_or_default() {
304 settings::ModelMode::Default => AnthropicModelMode::Default,
305 settings::ModelMode::Thinking { budget_tokens } => {
306 AnthropicModelMode::Thinking { budget_tokens }
307 }
308 };
309 let supports_thinking = matches!(
310 mode,
311 AnthropicModelMode::Thinking { .. } | AnthropicModelMode::AdaptiveThinking
312 );
313 let supports_adaptive_thinking = matches!(mode, AnthropicModelMode::AdaptiveThinking);
314
315 anthropic::Model {
316 display_name: available
317 .display_name
318 .clone()
319 .unwrap_or_else(|| available.name.clone()),
320 id: available.name.clone(),
321 max_input_tokens: available.max_tokens,
322 max_output_tokens: available.max_output_tokens.unwrap_or(4_096),
323 default_temperature: available.default_temperature.unwrap_or(1.0),
324 mode,
325 supports_thinking,
326 supports_adaptive_thinking,
327 supports_images: true,
328 supports_speed: false,
329 supported_effort_levels: if supports_adaptive_thinking {
330 vec![
331 anthropic::Effort::Low,
332 anthropic::Effort::Medium,
333 anthropic::Effort::High,
334 anthropic::Effort::XHigh,
335 anthropic::Effort::Max,
336 ]
337 } else {
338 vec![]
339 },
340 tool_override: available.tool_override.clone(),
341 extra_beta_headers: available.extra_beta_headers.clone(),
342 }
343}
344
237pub struct AnthropicModel {345pub struct AnthropicModel {
238 id: LanguageModelId,346 id: LanguageModelId,
239 model: anthropic::Model,347 model: anthropic::Model,
@@ -288,7 +396,7 @@ impl LanguageModel for AnthropicModel { · 48 unmodified lines
288 }396 }
289 397
290 fn name(&self) -> LanguageModelName {398 fn name(&self) -> LanguageModelName {
291 LanguageModelName::from(self.model.display_name().to_string())399 LanguageModelName::from(self.model.display_name.clone())
292 }400 }
293 401
294 fn provider_id(&self) -> LanguageModelProviderId {402 fn provider_id(&self) -> LanguageModelProviderId {
@@ -304,7 +412,7 @@ impl LanguageModel for AnthropicModel { · 9 unmodified lines
304 }412 }
305 413
306 fn supports_images(&self) -> bool {414 fn supports_images(&self) -> bool {
307 true415 self.model.supports_images
308 }416 }
309 417
310 fn supports_streaming_tools(&self) -> bool {418 fn supports_streaming_tools(&self) -> bool {
@@ -320,44 +428,37 @@ impl LanguageModel for AnthropicModel { · 9 unmodified lines
320 }428 }
321 429
322 fn supports_thinking(&self) -> bool {430 fn supports_thinking(&self) -> bool {
323 self.model.supports_thinking()431 self.model.supports_thinking
324 }432 }
325 433
326 fn supports_fast_mode(&self) -> bool {434 fn supports_fast_mode(&self) -> bool {
327 self.model.supports_speed()435 self.model.supports_speed
328 }436 }
329 437
330 fn supported_effort_levels(&self) -> Vec<language_model::LanguageModelEffortLevel> {438 fn supported_effort_levels(&self) -> Vec<language_model::LanguageModelEffortLevel> {
331 if self.model.supports_adaptive_thinking() {439 self.model
332 vec![440 .supported_effort_levels
333 language_model::LanguageModelEffortLevel {441 .iter()
334 name: "Low".into(),442 .map(|e| {
335 value: "low".into(),443 let is_default = matches!(e, anthropic::Effort::High);
336 is_default: false,444 let (name, value) = match e {
337 },445 anthropic::Effort::Low => ("Low".into(), "low".into()),
338 language_model::LanguageModelEffortLevel {446 anthropic::Effort::Medium => ("Medium".into(), "medium".into()),
339 name: "Medium".into(),447 anthropic::Effort::High => ("High".into(), "high".into()),
340 value: "medium".into(),448 anthropic::Effort::XHigh => ("XHigh".into(), "xhigh".into()),
341 is_default: false,449 anthropic::Effort::Max => ("Max".into(), "max".into()),
342 },450 };
343 language_model::LanguageModelEffortLevel {
344 name: "High".into(),
345 value: "high".into(),
346 is_default: true,
347 },
348 language_model::LanguageModelEffortLevel {451 language_model::LanguageModelEffortLevel {
349 name: "Max".into(),452 name,
350 value: "max".into(),453 value,
351 is_default: false,454 is_default,
352 },455 }
353 ]456 })
354 } else {457 .collect::<Vec<_>>()
355 Vec::new()
356 }
357 }458 }
358 459
359 fn telemetry_id(&self) -> String {460 fn telemetry_id(&self) -> String {
360 format!("anthropic/{}", self.model.id())461 format!("anthropic/{}", self.model.id)
361 }462 }
362 463
363 fn api_key(&self, cx: &App) -> Option<String> {464 fn api_key(&self, cx: &App) -> Option<String> {
@@ -368,11 +469,11 @@ impl LanguageModel for AnthropicModel { · 4 unmodified lines
368 }469 }
369 470
370 fn max_token_count(&self) -> u64 {471 fn max_token_count(&self) -> u64 {
371 self.model.max_token_count()472 self.model.max_input_tokens
372 }473 }
373 474
374 fn max_output_tokens(&self) -> Option<u64> {475 fn max_output_tokens(&self) -> Option<u64> {
375 Some(self.model.max_output_tokens())476 Some(self.model.max_output_tokens)
376 }477 }
377 478
378 fn stream_completion(479 fn stream_completion(
@@ -386,14 +487,16 @@ impl LanguageModel for AnthropicModel { · 7 unmodified lines
386 LanguageModelCompletionError,487 LanguageModelCompletionError,
387 >,488 >,
388 > {489 > {
490 let has_tools = !request.tools.is_empty();
491 let request_id = self.model.request_id(has_tools).to_string();
389 let mut request = into_anthropic(492 let mut request = into_anthropic(
390 request,493 request,
391 self.model.request_id().into(),494 request_id,
392 self.model.default_temperature(),495 self.model.default_temperature,
393 self.model.max_output_tokens(),496 self.model.max_output_tokens,
394 self.model.mode(),497 self.model.mode.clone(),
395 );498 );
396 if !self.model.supports_speed() {499 if !self.model.supports_speed {
397 request.speed = None;500 request.speed = None;
398 }501 }
399 let request = self.stream_completion(request, cx);502 let request = self.stream_completion(request, cx);
@@ -405,13 +508,7 @@ impl LanguageModel for AnthropicModel { · 5 unmodified lines
405 }508 }
406 509
407 fn cache_configuration(&self) -> Option<LanguageModelCacheConfiguration> {510 fn cache_configuration(&self) -> Option<LanguageModelCacheConfiguration> {
408 self.model511 None
409 .cache_configuration()
410 .map(|config| LanguageModelCacheConfiguration {
411 max_cache_anchors: config.max_cache_anchors,
412 should_speculate: config.should_speculate,
413 min_total_token: config.min_total_token,
414 })
415 }512 }
416}513}
417 514
docs/src/ai/llm-providers.md+0−5
@@ -181,11 +181,6 @@ You can add custom models to the Anthropic provider by adding the following to y · 180 unmodified lines
181 "display_name": "Sonnet 2024-June",181 "display_name": "Sonnet 2024-June",
182 "max_tokens": 128000,182 "max_tokens": 128000,
183 "max_output_tokens": 2560,183 "max_output_tokens": 2560,
184 "cache_configuration": {
185 "max_cache_anchors": 10,
186 "min_total_token": 10000,
187 "should_speculate": false
188 },
189 "tool_override": "some-model-that-supports-toolcalling"184 "tool_override": "some-model-that-supports-toolcalling"
190 }185 }
191 ]186 ]