History

Surface upstream rate limits from Anthropic (#16118)

This PR makes it so hitting upstream rate limits from Anthropic result in an HTTP 429 response instead of an HTTP 500. To do this we need to surface structured errors out of the `anthropic` crate. Release Notes: - N/A

Marshall Bowers committed 2 years ago

Commit ebdb7551 parent fbb533b6 files · +145 −48Browse files

Unified · Split

6 files
  1. Cargo.lock +1 −0
  2. crates/anthropic/Cargo.toml +1 −0
  3. crates/anthropic/src/anthropic.rs +102 −38
  4. crates/collab/src/llm.rs +14 −1
  5. crates/language_model/src/provider/anthropic.rs +15 −5
  6. crates/language_model/src/provider/cloud.rs +12 −4
Cargo.lock+1−0
@@ -255,6 +255,7 @@ dependencies = [ · 254 unmodified lines
255255 "serde",
256256 "serde_json",
257257 "strum",
258+ "thiserror",
258259 "tokio",
259260 ]
260261
crates/anthropic/Cargo.toml+1−0
@@ -24,6 +24,7 @@ schemars = { workspace = true, optional = true } · 23 unmodified lines
2424 serde.workspace = true
2525 serde_json.workspace = true
2626 strum.workspace = true
27+thiserror.workspace = true
2728
2829 [dev-dependencies]
2930 tokio.workspace = true
crates/anthropic/src/anthropic.rs+102−38
@@ -1,12 +1,14 @@
11 mod supported_countries;
22
3−use anyhow::{anyhow, Result};
3+use anyhow::{anyhow, Context, Result};
44 use futures::{io::BufReader, stream::BoxStream, AsyncBufReadExt, AsyncReadExt, Stream, StreamExt};
55 use http_client::{AsyncBody, HttpClient, Method, Request as HttpRequest};
66 use isahc::config::Configurable;
77 use serde::{Deserialize, Serialize};
8+use std::str::FromStr;
89 use std::time::Duration;
9−use strum::EnumIter;
10+use strum::{EnumIter, EnumString};
11+use thiserror::Error;
1012
1113 pub use supported_countries::*;
1214
@@ -96,7 +98,7 @@ pub async fn complete( · 83 unmodified lines
9698 api_url: &str,
9799 api_key: &str,
98100 request: Request,
99−) -> Result<Response> {
101+) -> Result<Response, AnthropicError> {
100102 let uri = format!("{api_url}/v1/messages");
101103 let request_builder = HttpRequest::builder()
102104 .method(Method::POST)
@@ -106,24 +108,40 @@ pub async fn complete( · 3 unmodified lines
106108 .header("X-Api-Key", api_key)
107109 .header("Content-Type", "application/json");
108110
109− let serialized_request = serde_json::to_string(&request)?;
110− let request = request_builder.body(AsyncBody::from(serialized_request))?;
111+ let serialized_request =
112+ serde_json::to_string(&request).context("failed to serialize request")?;
113+ let request = request_builder
114+ .body(AsyncBody::from(serialized_request))
115+ .context("failed to construct request body")?;
111116
112− let mut response = client.send(request).await?;
117+ let mut response = client
118+ .send(request)
119+ .await
120+ .context("failed to send request to Anthropic")?;
113121 if response.status().is_success() {
114122 let mut body = Vec::new();
115− response.body_mut().read_to_end(&mut body).await?;
116− let response_message: Response = serde_json::from_slice(&body)?;
123+ response
124+ .body_mut()
125+ .read_to_end(&mut body)
126+ .await
127+ .context("failed to read response body")?;
128+ let response_message: Response =
129+ serde_json::from_slice(&body).context("failed to deserialize response body")?;
117130 Ok(response_message)
118131 } else {
119132 let mut body = Vec::new();
120− response.body_mut().read_to_end(&mut body).await?;
121− let body_str = std::str::from_utf8(&body)?;
122− Err(anyhow!(
133+ response
134+ .body_mut()
135+ .read_to_end(&mut body)
136+ .await
137+ .context("failed to read response body")?;
138+ let body_str =
139+ std::str::from_utf8(&body).context("failed to parse response body as UTF-8")?;
140+ Err(AnthropicError::Other(anyhow!(
123141 "Failed to connect to API: {} {}",
124142 response.status(),
125143 body_str
126− ))
144+ )))
127145 }
128146 }
129147
@@ -133,7 +151,7 @@ pub async fn stream_completion( · 3 unmodified lines
133151 api_key: &str,
134152 request: Request,
135153 low_speed_timeout: Option<Duration>,
136−) -> Result<BoxStream<'static, Result<Event>>> {
154+) -> Result<BoxStream<'static, Result<Event, AnthropicError>>, AnthropicError> {
137155 let request = StreamingRequest {
138156 base: request,
139157 stream: true,
@@ -149,10 +167,16 @@ pub async fn stream_completion( · 9 unmodified lines
149167 if let Some(low_speed_timeout) = low_speed_timeout {
150168 request_builder = request_builder.low_speed_timeout(100, low_speed_timeout);
151169 }
152− let serialized_request = serde_json::to_string(&request)?;
153− let request = request_builder.body(AsyncBody::from(serialized_request))?;
154−
155− let mut response = client.send(request).await?;
170+ let serialized_request =
171+ serde_json::to_string(&request).context("failed to serialize request")?;
172+ let request = request_builder
173+ .body(AsyncBody::from(serialized_request))
174+ .context("failed to construct request body")?;
175+
176+ let mut response = client
177+ .send(request)
178+ .await
179+ .context("failed to send request to Anthropic")?;
156180 if response.status().is_success() {
157181 let reader = BufReader::new(response.into_body());
158182 Ok(reader
@@ -163,36 +187,41 @@ pub async fn stream_completion( · 4 unmodified lines
163187 let line = line.strip_prefix("data: ")?;
164188 match serde_json::from_str(line) {
165189 Ok(response) => Some(Ok(response)),
166− Err(error) => Some(Err(anyhow!(error))),
190+ Err(error) => Some(Err(AnthropicError::Other(anyhow!(error)))),
167191 }
168192 }
169− Err(error) => Some(Err(anyhow!(error))),
193+ Err(error) => Some(Err(AnthropicError::Other(anyhow!(error)))),
170194 }
171195 })
172196 .boxed())
173197 } else {
174198 let mut body = Vec::new();
175− response.body_mut().read_to_end(&mut body).await?;
199+ response
200+ .body_mut()
201+ .read_to_end(&mut body)
202+ .await
203+ .context("failed to read response body")?;
176204
177− let body_str = std::str::from_utf8(&body)?;
205+ let body_str =
206+ std::str::from_utf8(&body).context("failed to parse response body as UTF-8")?;
178207
179208 match serde_json::from_str::<Event>(body_str) {
180− Ok(Event::Error { error }) => Err(api_error_to_err(error)),
181− Ok(_) => Err(anyhow!(
209+ Ok(Event::Error { error }) => Err(AnthropicError::ApiError(error)),
210+ Ok(_) => Err(AnthropicError::Other(anyhow!(
182211 "Unexpected success response while expecting an error: '{body_str}'",
183− )),
184− Err(_) => Err(anyhow!(
212+ ))),
213+ Err(_) => Err(AnthropicError::Other(anyhow!(
185214 "Failed to connect to API: {} {}",
186215 response.status(),
187216 body_str,
188− )),
217+ ))),
189218 }
190219 }
191220 }
192221
193222 pub fn extract_text_from_events(
194− response: impl Stream<Item = Result<Event>>,
195−) -> impl Stream<Item = Result<String>> {
223+ response: impl Stream<Item = Result<Event, AnthropicError>>,
224+) -> impl Stream<Item = Result<String, AnthropicError>> {
196225 response.filter_map(|response| async move {
197226 match response {
198227 Ok(response) => match response {
@@ -204,7 +233,7 @@ pub fn extract_text_from_events( · 5 unmodified lines
204233 ContentDelta::TextDelta { text } => Some(Ok(text)),
205234 _ => None,
206235 },
207− Event::Error { error } => Some(Err(api_error_to_err(error))),
236+ Event::Error { error } => Some(Err(AnthropicError::ApiError(error))),
208237 _ => None,
209238 },
210239 Err(error) => Some(Err(error)),
@@ -212,15 +241,6 @@ pub fn extract_text_from_events( · 1 unmodified line
212241 })
213242 }
214243
215−fn api_error_to_err(
216− ApiError {
217− error_type,
218− message,
219− }: ApiError,
220−) -> anyhow::Error {
221− anyhow!("API error. Type: '{error_type}', message: '{message}'",)
222−}
223−
224244 #[derive(Debug, Serialize, Deserialize)]
225245 pub struct Message {
226246 pub role: Role,
@@ -374,9 +394,53 @@ pub struct MessageDelta { · 147 unmodified lines
374394 pub stop_sequence: Option<String>,
375395 }
376396
397+#[derive(Error, Debug)]
398+pub enum AnthropicError {
399+ #[error("an error occurred while interacting with the Anthropic API: {error_type}: {message}", error_type = .0.error_type, message = .0.message)]
400+ ApiError(ApiError),
401+ #[error("{0}")]
402+ Other(#[from] anyhow::Error),
403+}
404+
377405 #[derive(Debug, Serialize, Deserialize)]
378406 pub struct ApiError {
379407 #[serde(rename = "type")]
380408 pub error_type: String,
381409 pub message: String,
382410 }
411+
412+/// An Anthropic API error code.
413+/// https://docs.anthropic.com/en/api/errors#http-errors
414+#[derive(Debug, PartialEq, Eq, Clone, Copy, EnumString)]
415+#[strum(serialize_all = "snake_case")]
416+pub enum ApiErrorCode {
417+ /// 400 - `invalid_request_error`: There was an issue with the format or content of your request.
418+ InvalidRequestError,
419+ /// 401 - `authentication_error`: There's an issue with your API key.
420+ AuthenticationError,
421+ /// 403 - `permission_error`: Your API key does not have permission to use the specified resource.
422+ PermissionError,
423+ /// 404 - `not_found_error`: The requested resource was not found.
424+ NotFoundError,
425+ /// 413 - `request_too_large`: Request exceeds the maximum allowed number of bytes.
426+ RequestTooLarge,
427+ /// 429 - `rate_limit_error`: Your account has hit a rate limit.
428+ RateLimitError,
429+ /// 500 - `api_error`: An unexpected error has occurred internal to Anthropic's systems.
430+ ApiError,
431+ /// 529 - `overloaded_error`: Anthropic's API is temporarily overloaded.
432+ OverloadedError,
433+}
434+
435+impl ApiError {
436+ pub fn code(&self) -> Option<ApiErrorCode> {
437+ ApiErrorCode::from_str(&self.error_type).ok()
438+ }
439+
440+ pub fn is_rate_limit_error(&self) -> bool {
441+ match self.error_type.as_str() {
442+ "rate_limit_error" => true,
443+ _ => false,
444+ }
445+ }
446+}
crates/collab/src/llm.rs+14−1
@@ -197,7 +197,20 @@ async fn perform_completion( · 196 unmodified lines
197197 request,
198198 None,
199199 )
200− .await?;
200+ .await
201+ .map_err(|err| match err {
202+ anthropic::AnthropicError::ApiError(ref api_error) => {
203+ if api_error.code() == Some(anthropic::ApiErrorCode::RateLimitError) {
204+ return Error::http(
205+ StatusCode::TOO_MANY_REQUESTS,
206+ "Upstream Anthropic rate limit exceeded.".to_string(),
207+ );
208+ }
209+
210+ Error::Internal(anyhow!(err))
211+ }
212+ anthropic::AnthropicError::Other(err) => Error::Internal(err),
213+ })?;
201214
202215 chunks
203216 .map(move |event| {
crates/language_model/src/provider/anthropic.rs+15−5
@@ -3,6 +3,7 @@ use crate::{ · 2 unmodified lines
33 LanguageModelProvider, LanguageModelProviderId, LanguageModelProviderName,
44 LanguageModelProviderState, LanguageModelRequest, RateLimiter, Role,
55 };
6+use anthropic::AnthropicError;
67 use anyhow::{anyhow, Context as _, Result};
78 use collections::BTreeMap;
89 use editor::{Editor, EditorElement, EditorStyle};
@@ -259,7 +260,9 @@ impl AnthropicModel { · 250 unmodified lines
259260
260261 async move {
261262 let api_key = api_key.ok_or_else(|| anyhow!("missing api key"))?;
262− anthropic::complete(http_client.as_ref(), &api_url, &api_key, request).await
263+ anthropic::complete(http_client.as_ref(), &api_url, &api_key, request)
264+ .await
265+ .context("failed to retrieve completion")
263266 }
264267 .boxed()
265268 }
@@ -268,7 +271,8 @@ impl AnthropicModel { · 2 unmodified lines
268271 &self,
269272 request: anthropic::Request,
270273 cx: &AsyncAppContext,
271− ) -> BoxFuture<'static, Result<BoxStream<'static, Result<anthropic::Event>>>> {
274+ ) -> BoxFuture<'static, Result<BoxStream<'static, Result<anthropic::Event, AnthropicError>>>>
275+ {
272276 let http_client = self.http_client.clone();
273277
274278 let Ok((api_key, api_url, low_speed_timeout)) = cx.read_model(&self.state, |state, cx| {
@@ -291,7 +295,7 @@ impl AnthropicModel { · 16 unmodified lines
291295 request,
292296 low_speed_timeout,
293297 );
294− request.await
298+ request.await.context("failed to stream completion")
295299 }
296300 .boxed()
297301 }
@@ -338,10 +342,16 @@ impl LanguageModel for AnthropicModel { · 40 unmodified lines
338342 let request = request.into_anthropic(self.model.id().into());
339343 let request = self.stream_completion(request, cx);
340344 let future = self.request_limiter.stream(async move {
341− let response = request.await?;
345+ let response = request.await.map_err(|err| anyhow!(err))?;
342346 Ok(anthropic::extract_text_from_events(response))
343347 });
344− async move { Ok(future.await?.boxed()) }.boxed()
348+ async move {
349+ Ok(future
350+ .await?
351+ .map(|result| result.map_err(|err| anyhow!(err)))
352+ .boxed())
353+ }
354+ .boxed()
345355 }
346356
347357 fn use_any_tool(
crates/language_model/src/provider/cloud.rs+12−4
@@ -4,7 +4,8 @@ use crate::{ · 3 unmodified lines
44 LanguageModelName, LanguageModelProviderId, LanguageModelProviderName,
55 LanguageModelProviderState, LanguageModelRequest, RateLimiter, ZedModel,
66 };
7−use anyhow::{anyhow, bail, Result};
7+use anthropic::AnthropicError;
8+use anyhow::{anyhow, bail, Context as _, Result};
89 use client::{Client, PerformCompletionParams, UserStore, EXPIRED_LLM_TOKEN_HEADER_NAME};
910 use collections::BTreeMap;
1011 use feature_flags::{FeatureFlagAppExt, LanguageModels};
@@ -446,16 +447,23 @@ impl LanguageModel for CloudLanguageModel { · 435 unmodified lines
446447 match body.read_line(&mut buffer).await {
447448 Ok(0) => Ok(None),
448449 Ok(_) => {
449− let event: anthropic::Event = serde_json::from_str(&buffer)?;
450+ let event: anthropic::Event = serde_json::from_str(&buffer)
451+ .context("failed to parse Anthropic event")?;
450452 Ok(Some((event, body)))
451453 }
452− Err(e) => Err(e.into()),
454+ Err(err) => Err(AnthropicError::Other(err.into())),
453455 }
454456 });
455457
456458 Ok(anthropic::extract_text_from_events(stream))
457459 });
458− async move { Ok(future.await?.boxed()) }.boxed()
460+ async move {
461+ Ok(future
462+ .await?
463+ .map(|result| result.map_err(|err| anyhow!(err)))
464+ .boxed())
465+ }
466+ .boxed()
459467 }
460468 CloudModel::OpenAi(model) => {
461469 let client = self.client.clone();