History

fix(claude): report usage limits on retried turns (#10549)

Wout Stiens committed 2 months ago

Commit 3bf74eb1 parent 0d345792 files · +180 −2Browse files

Unified · Split

2 files
  1. apps/server/src/provider/Layers/ClaudeAdapter.test.ts +173 −1
  2. apps/server/src/provider/Layers/ClaudeAdapter.ts +7 −1
apps/server/src/provider/Layers/ClaudeAdapter.test.ts+173−1
@@ -2367,6 +2367,139 @@ describe("ClaudeAdapterLive", () => { · 2366 unmodified lines
23672367 );
23682368 });
23692369
2370+ const usageLimitMessage =
2371+ "Claude usage limit reached. Send the message again once the limit resets.";
2372+ const genericApiErrorMessage = "Claude gave up after repeated API errors.";
2373+ const rateLimitAssistant = {
2374+ type: "assistant",
2375+ session_id: "sdk-session-limit",
2376+ uuid: "assistant-limit",
2377+ parent_tool_use_id: null,
2378+ error: "rate_limit",
2379+ message: {
2380+ id: "assistant-message-limit",
2381+ model: "<synthetic>",
2382+ content: [{ type: "text", text: "You've hit your session limit" }],
2383+ },
2384+ };
2385+ const rateLimitResult = {
2386+ type: "result",
2387+ subtype: "success",
2388+ is_error: true,
2389+ terminal_reason: "api_error",
2390+ session_id: "sdk-session-limit",
2391+ uuid: "result-limit",
2392+ };
2393+
2394+ it.effect.each([
2395+ {
2396+ name: "an assistant-only rate limit",
2397+ messages: [rateLimitAssistant],
2398+ expected: usageLimitMessage,
2399+ },
2400+ {
2401+ name: "a normal parent response after a rate limit",
2402+ messages: [rateLimitAssistant, { ...rateLimitAssistant, error: undefined }],
2403+ expected: genericApiErrorMessage,
2404+ },
2405+ {
2406+ name: "a server error after a rate limit",
2407+ messages: [rateLimitAssistant, { ...rateLimitAssistant, error: "server_error" }],
2408+ expected: genericApiErrorMessage,
2409+ },
2410+ {
2411+ name: "a subagent rate limit",
2412+ messages: [{ ...rateLimitAssistant, parent_tool_use_id: "nested-tool" }],
2413+ expected: genericApiErrorMessage,
2414+ },
2415+ {
2416+ name: "a subagent response after a parent rate limit",
2417+ messages: [
2418+ rateLimitAssistant,
2419+ { ...rateLimitAssistant, error: undefined, parent_tool_use_id: "nested-tool" },
2420+ ],
2421+ expected: usageLimitMessage,
2422+ },
2423+ ])("classifies the terminal API failure after $name", ({ messages, expected }) => {
2424+ const harness = makeHarness();
2425+ return Effect.gen(function* () {
2426+ const adapter = yield* ClaudeAdapter;
2427+ const eventsFiber = yield* adapter.streamEvents.pipe(
2428+ Stream.takeUntil((event) => event.type === "turn.completed"),
2429+ Stream.runCollect,
2430+ Effect.forkChild,
2431+ );
2432+ const session = yield* adapter.startSession({
2433+ threadId: THREAD_ID,
2434+ provider: ProviderDriverKind.make("claudeAgent"),
2435+ runtimeMode: "full-access",
2436+ });
2437+ yield* adapter.sendTurn({ threadId: session.threadId, input: "hello", attachments: [] });
2438+ for (const [index, message] of messages.entries()) {
2439+ harness.query.emit({ ...message, uuid: `assistant-${index}` } as unknown as SDKMessage);
2440+ }
2441+ harness.query.emit(rateLimitResult as unknown as SDKMessage);
2442+
2443+ const events = Array.from(yield* Fiber.join(eventsFiber));
2444+ const errors = events.filter((event) => event.type === "runtime.error");
2445+ assert.equal(errors.length, 1);
2446+ assert.equal(errors[0]?.payload.message, expected);
2447+ assert.equal(completedTurn(events).state, "failed");
2448+ assert.equal(completedTurn(events).errorMessage, expected);
2449+ }).pipe(
2450+ Effect.provideService(Random.Random, makeDeterministicRandomService()),
2451+ Effect.provide(harness.layer),
2452+ );
2453+ });
2454+
2455+ it.effect("names repeated usage limits without carrying them into a later turn", () => {
2456+ const harness = makeHarness();
2457+ return Effect.gen(function* () {
2458+ const adapter = yield* ClaudeAdapter;
2459+ const session = yield* adapter.startSession({
2460+ threadId: THREAD_ID,
2461+ provider: ProviderDriverKind.make("claudeAgent"),
2462+ runtimeMode: "full-access",
2463+ });
2464+ for (const [index, expected] of [
2465+ usageLimitMessage,
2466+ usageLimitMessage,
2467+ genericApiErrorMessage,
2468+ ].entries()) {
2469+ const eventsFiber = yield* adapter.streamEvents.pipe(
2470+ Stream.takeUntil((event) => event.type === "turn.completed"),
2471+ Stream.runCollect,
2472+ Effect.forkChild,
2473+ );
2474+ yield* adapter.sendTurn({ threadId: session.threadId, input: "again", attachments: [] });
2475+ if (index === 0) {
2476+ harness.query.emit({
2477+ type: "rate_limit_event",
2478+ rate_limit_info: { status: "rejected", rateLimitType: "five_hour" },
2479+ session_id: "sdk-session-limit",
2480+ uuid: "limit-rejected",
2481+ } as unknown as SDKMessage);
2482+ }
2483+ if (index < 2) {
2484+ harness.query.emit({
2485+ ...rateLimitAssistant,
2486+ uuid: `assistant-limit-${index}`,
2487+ } as unknown as SDKMessage);
2488+ }
2489+ harness.query.emit({
2490+ ...rateLimitResult,
2491+ uuid: `result-limit-${index}`,
2492+ } as unknown as SDKMessage);
2493+ const payload = completedTurn(Array.from(yield* Fiber.join(eventsFiber)));
2494+ assert.equal(payload.state, "failed");
2495+ assert.equal(payload.errorMessage, expected);
2496+ }
2497+ }).pipe(
2498+ Effect.provideService(Random.Random, makeDeterministicRandomService()),
2499+ Effect.provide(harness.layer),
2500+ );
2501+ });
2502+
23702503 it.effect.each([
23712504 {
23722505 name: "listed error with api_error",
@@ -2418,6 +2551,43 @@ describe("ClaudeAdapterLive", () => { · 45 unmodified lines
24182551 expected: /repeated API errors/,
24192552 expectedState: "failed",
24202553 },
2554+ {
2555+ name: "assistant rate limit followed by overload",
2556+ evidence: "assistant-rate-limit",
2557+ result: {
2558+ subtype: "success",
2559+ is_error: true,
2560+ terminal_reason: "api_error",
2561+ api_error_status: 529,
2562+ errors: [],
2563+ },
2564+ expected: /overloaded \(529\)/,
2565+ expectedState: "failed",
2566+ },
2567+ {
2568+ name: "assistant rate limit followed by a listed error",
2569+ evidence: "assistant-rate-limit",
2570+ result: {
2571+ subtype: "success",
2572+ is_error: true,
2573+ terminal_reason: "api_error",
2574+ errors: ["Tool execution failed: EACCES"],
2575+ },
2576+ expected: /EACCES/,
2577+ expectedState: "failed",
2578+ },
2579+ {
2580+ name: "assistant rate limit followed by an interrupt",
2581+ evidence: "assistant-rate-limit",
2582+ result: {
2583+ subtype: "error_during_execution",
2584+ is_error: true,
2585+ terminal_reason: "aborted_tools",
2586+ errors: [],
2587+ },
2588+ expected: undefined,
2589+ expectedState: "interrupted",
2590+ },
24212591 ...[
24222592 "recovered-missing-reset",
24232593 "recovered-next-reset",
@@ -2465,7 +2635,9 @@ describe("ClaudeAdapterLive", () => { · 41 unmodified lines
24652635 input: "synthetic hello",
24662636 attachments: [],
24672637 });
2468− if (evidence === "auth" || evidence === "nested-auth") {
2638+ if (evidence === "assistant-rate-limit") {
2639+ harness.query.emit(rateLimitAssistant as unknown as SDKMessage);
2640+ } else if (evidence === "auth" || evidence === "nested-auth") {
24692641 harness.query.emit({
24702642 type: "assistant",
24712643 session_id: "sdk-audit",
apps/server/src/provider/Layers/ClaudeAdapter.ts+7−1
@@ -161,6 +161,7 @@ interface ClaudeTurnState { · 160 unmodified lines
161161 nextSyntheticAssistantBlockIndex: number;
162162 authenticationFailureMessage: string | undefined;
163163 rejectedRateLimitTypes: Set<string>;
164+ latestAssistantRateLimited: boolean;
164165 }
165166
166167 interface AssistantTextBlockState {
@@ -3168,6 +3169,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( · 3001 unmodified lines
31683169 nextSyntheticAssistantBlockIndex: -1,
31693170 authenticationFailureMessage: undefined,
31703171 rejectedRateLimitTypes: new Set(),
3172+ latestAssistantRateLimited: false,
31713173 };
31723174 context.session = {
31733175 ...context.session,
@@ -3226,6 +3228,9 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( · 52 unmodified lines
32263228 }
32273229
32283230 if (context.turnState) {
3231+ // Limited retries may only carry an assistant error, without a new window
3232+ // event. Later parent responses replace this evidence if the turn recovers.
3233+ context.turnState.latestAssistantRateLimited = message.error === "rate_limit";
32293234 // The CLI can report authentication failure before ending the turn as a
32303235 // generic API error, so retain that evidence for the result fallback.
32313236 if (message.error === "authentication_failed") {
@@ -3263,7 +3268,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( · 31 unmodified lines
32633268 const turn = context.turnState;
32643269 const failureHint =
32653270 turn?.authenticationFailureMessage ??
3266− (turn && turn.rejectedRateLimitTypes.size > 0
3271+ (turn && (turn.rejectedRateLimitTypes.size > 0 || turn.latestAssistantRateLimited)
32673272 ? "Claude usage limit reached. Send the message again once the limit resets."
32683273 : undefined);
32693274 const { status, errorMessage } = resultOutcome(message, failureHint);
@@ -4949,6 +4954,7 @@ export const makeClaudeAdapter = Effect.fn("makeClaudeAdapter")(function* ( · 1679 unmodified lines
49494954 nextSyntheticAssistantBlockIndex: -1,
49504955 authenticationFailureMessage: undefined,
49514956 rejectedRateLimitTypes: new Set(),
4957+ latestAssistantRateLimited: false,
49524958 };
49534959
49544960 const updatedAt = yield* nowIso;