History

fix(xai): remap bad-credentials 403s to unauthorized

- normalize xAI `403` bad-credentials responses to `401` before conductor retry handling - preserve websocket error headers while applying the normalized status/retry hints - share bad-credentials detection across flat and nested payload shapes for HTTP/websocket paths

Luis Pater committed 3 months ago

Commit dd67f561 parent 0a95fa64 files · +130 −5Browse files

Unified · Split

4 files
  1. internal/runtime/executor/xai_executor_response.go +38 −5
  2. internal/runtime/executor/xai_status_err_test.go +52 −0
  3. internal/runtime/executor/xai_websockets_executor.go +3 −0
  4. internal/runtime/executor/xai_websockets_executor_test.go +37 −0
internal/runtime/executor/xai_executor_response.go+38−5
@@ -857,13 +857,25 @@ func xaiPatchCompletedOutput(eventData []byte, outputItemsByIndex map[int64][]by · 856 unmodified lines
857857 // cli-chat-proxy ("Usage resets over a rolling 24-hour window").
858858 const xaiFreeUsageExhaustedCooldown = 24 * time.Hour
859859
860−// xaiStatusErr wraps upstream error bodies so free-tier exhaustion
861−// (subscription:free-usage-exhausted) carries a 24h RetryAfter hint for
862−// auth cooldown / account rotation. Generic 429s stay without an explicit
863−// retry hint so conductor backoff still applies.
860+// xaiStatusErr normalizes upstream xAI error bodies for conductor behavior:
861+// - credential invalidation (403 bad-credentials) is remapped to 401 so the
862+// existing OAuth refresh-once-and-retry path runs instead of payment cooldown
863+// - free-tier exhaustion (subscription:free-usage-exhausted) carries a 24h
864+// RetryAfter hint for auth cooldown / account rotation
865+//
866+// Generic 429s stay without an explicit retry hint so conductor backoff still applies.
864867 func xaiStatusErr(code int, body []byte) statusErr {
865868 err := statusErr{code: code, msg: string(body)}
866− if code != http.StatusTooManyRequests || len(body) == 0 {
869+ if len(body) == 0 {
870+ return err
871+ }
872+ if code == http.StatusForbidden && isXAIBadCredentialsBody(body) {
873+ // Upstream returns 403 for invalidated OAuth access tokens. Map to 401 so
874+ // tryRefreshAfterUnauthorized / MarkResult unauthorized handling applies.
875+ err.code = http.StatusUnauthorized
876+ return err
877+ }
878+ if code != http.StatusTooManyRequests {
867879 return err
868880 }
869881 codeStr := strings.ToLower(gjson.GetBytes(body, "code").String())
@@ -879,3 +891,24 @@ func xaiStatusErr(code int, body []byte) statusErr { · 9 unmodified lines
879891 }
880892 return err
881893 }
894+
895+// isXAIBadCredentialsBody reports whether an xAI error body indicates an
896+// invalidated/unusable OAuth access token rather than a generic permission or
897+// payment failure. HTTP and websocket payloads both use this helper, so nested
898+// error.code / error.message shapes are checked as well as flat bodies.
899+func isXAIBadCredentialsBody(body []byte) bool {
900+ for _, path := range []string{"code", "error.code", "body.error.code"} {
901+ if strings.Contains(strings.ToLower(gjson.GetBytes(body, path).String()), "bad-credentials") {
902+ return true
903+ }
904+ }
905+ for _, path := range []string{"error", "error.message", "message", "body.error", "body.error.message"} {
906+ msg := strings.ToLower(gjson.GetBytes(body, path).String())
907+ if strings.Contains(msg, "access token could not be validated") {
908+ return true
909+ }
910+ }
911+ raw := strings.ToLower(string(body))
912+ return strings.Contains(raw, "bad-credentials") ||
913+ strings.Contains(raw, "access token could not be validated")
914+}
internal/runtime/executor/xai_status_err_test.go+52−0
@@ -2,6 +2,7 @@ package executor · 1 unmodified line
22
33 import (
44 "net/http"
5+ "strings"
56 "testing"
67 "time"
78 )
@@ -31,7 +32,58 @@ func TestXAIStatusErr_Generic429HasNoRetryAfter(t *testing.T) { · 23 unmodified lines
3132 func TestXAIStatusErr_Non429Unchanged(t *testing.T) {
3233 body := []byte(`{"error":"nope"}`)
3334 err := xaiStatusErr(http.StatusBadRequest, body)
35+ if err.StatusCode() != http.StatusBadRequest {
36+ t.Fatalf("status = %d, want 400", err.StatusCode())
37+ }
3438 if err.RetryAfter() != nil {
3539 t.Fatalf("expected nil RetryAfter for 400, got %v", *err.RetryAfter())
3640 }
3741 }
42+
43+func TestXAIStatusErr_BadCredentials403RemapsToUnauthorized(t *testing.T) {
44+ body := []byte(`{"code":"unauthenticated:bad-credentials","error":"The OAuth2 access token could not be validated."}`)
45+ err := xaiStatusErr(http.StatusForbidden, body)
46+ if err.StatusCode() != http.StatusUnauthorized {
47+ t.Fatalf("status = %d, want 401", err.StatusCode())
48+ }
49+ if !strings.Contains(err.Error(), "bad-credentials") {
50+ t.Fatalf("error body should be preserved, got %q", err.Error())
51+ }
52+ if err.RetryAfter() != nil {
53+ t.Fatalf("expected nil RetryAfter for bad-credentials, got %v", *err.RetryAfter())
54+ }
55+}
56+
57+func TestXAIStatusErr_BadCredentialsByMessageOnly(t *testing.T) {
58+ body := []byte(`{"error":"The OAuth2 access token could not be validated."}`)
59+ err := xaiStatusErr(http.StatusForbidden, body)
60+ if err.StatusCode() != http.StatusUnauthorized {
61+ t.Fatalf("status = %d, want 401", err.StatusCode())
62+ }
63+}
64+
65+func TestXAIStatusErr_BadCredentialsNestedErrorCode(t *testing.T) {
66+ body := []byte(`{"type":"error","status":403,"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`)
67+ err := xaiStatusErr(http.StatusForbidden, body)
68+ if err.StatusCode() != http.StatusUnauthorized {
69+ t.Fatalf("status = %d, want 401", err.StatusCode())
70+ }
71+}
72+
73+func TestXAIStatusErr_Generic403Unchanged(t *testing.T) {
74+ body := []byte(`{"code":"permission_denied","error":"model access is not allowed for this account"}`)
75+ err := xaiStatusErr(http.StatusForbidden, body)
76+ if err.StatusCode() != http.StatusForbidden {
77+ t.Fatalf("status = %d, want 403", err.StatusCode())
78+ }
79+ if err.RetryAfter() != nil {
80+ t.Fatalf("expected nil RetryAfter for generic 403, got %v", *err.RetryAfter())
81+ }
82+}
83+
84+func TestXAIStatusErr_EmptyBodyForbiddenUnchanged(t *testing.T) {
85+ err := xaiStatusErr(http.StatusForbidden, nil)
86+ if err.StatusCode() != http.StatusForbidden {
87+ t.Fatalf("status = %d, want 403", err.StatusCode())
88+ }
89+}
internal/runtime/executor/xai_websockets_executor.go+3−0
@@ -982,6 +982,9 @@ func parseXAIWebsocketError(payload []byte) (error, bool) { · 981 unmodified lines
982982 if wsErr, ok := parseCodexWebsocketError(payload); ok {
983983 if statusError, okStatus := wsErr.(statusErrWithHeaders); okStatus {
984984 xaiError := xaiStatusErr(statusError.code, payload)
985+ // Apply normalized status (e.g. 403 bad-credentials -> 401) and any
986+ // provider-specific retry hint while preserving websocket headers.
987+ statusError.code = xaiError.code
985988 if xaiError.retryAfter != nil {
986989 statusError.retryAfter = xaiError.retryAfter
987990 }
internal/runtime/executor/xai_websockets_executor_test.go+37−0
@@ -1600,6 +1600,43 @@ func TestParseXAIWebsocketErrorFreeUsageExhaustedSetsRetryAfter(t *testing.T) { · 1599 unmodified lines
16001600 }
16011601 }
16021602
1603+func TestParseXAIWebsocketErrorBadCredentialsRemapsToUnauthorized(t *testing.T) {
1604+ payload := []byte(`{"type":"error","status":403,"headers":{"x-request-id":"req-bad-credentials"},"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`)
1605+ err, ok := parseXAIWebsocketError(payload)
1606+ if !ok {
1607+ t.Fatal("expected xAI websocket error")
1608+ }
1609+
1610+ status, okStatus := err.(interface{ StatusCode() int })
1611+ if !okStatus || status.StatusCode() != http.StatusUnauthorized {
1612+ t.Fatalf("status = %#v, want 401", err)
1613+ }
1614+ headerSource, okHeaders := err.(interface{ Headers() http.Header })
1615+ if !okHeaders {
1616+ t.Fatalf("expected websocket error to preserve headers, got %#v", err)
1617+ }
1618+ if got := headerSource.Headers().Get("x-request-id"); got != "req-bad-credentials" {
1619+ t.Fatalf("x-request-id = %q, want req-bad-credentials", got)
1620+ }
1621+ parsed := gjson.Parse(err.Error())
1622+ if got := parsed.Get("error.code").String(); got != "unauthenticated:bad-credentials" {
1623+ t.Fatalf("error code = %q, want unauthenticated:bad-credentials; payload=%s", got, err)
1624+ }
1625+}
1626+
1627+func TestParseXAIWebsocketBareErrorBadCredentialsRemapsToUnauthorized(t *testing.T) {
1628+ payload := []byte(`{"status":403,"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`)
1629+ err, ok := parseXAIWebsocketError(payload)
1630+ if !ok {
1631+ t.Fatal("expected bare xAI websocket error")
1632+ }
1633+
1634+ status, okStatus := err.(interface{ StatusCode() int })
1635+ if !okStatus || status.StatusCode() != http.StatusUnauthorized {
1636+ t.Fatalf("status = %#v, want 401", err)
1637+ }
1638+}
1639+
16031640 func TestParseXAIWebsocketBareErrorFreeUsageExhaustedSetsRetryAfter(t *testing.T) {
16041641 payload := []byte(`{"status":429,"error":{"code":"subscription:free-usage-exhausted","message":"You've used all the included free usage for now."}}`)
16051642 err, ok := parseXAIWebsocketError(payload)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t