History

fix(xai): remap bad-credentials 403s to unauthorized

- normalize xAI `403` bad-credentials responses to `401` before conductor retry handling - preserve websocket error headers while applying the normalized status/retry hints - share bad-credentials detection across flat and nested payload shapes for HTTP/websocket paths

Luis Pater committed 3 months ago

Commit dd67f561 parent 0a95fa64 files · +130 −5Browse files

Unified · Split

4 files
  1. internal/runtime/executor/xai_executor_response.go +38 −5
  2. internal/runtime/executor/xai_status_err_test.go +52 −0
  3. internal/runtime/executor/xai_websockets_executor.go +3 −0
  4. internal/runtime/executor/xai_websockets_executor_test.go +37 −0
internal/runtime/executor/xai_executor_response.go+38−5
@@ -857,13 +857,25 @@ func xaiPatchCompletedOutput(eventData []byte, outputItemsByIndex map[int64][]by · 856 unmodified lines
857// cli-chat-proxy ("Usage resets over a rolling 24-hour window").857// cli-chat-proxy ("Usage resets over a rolling 24-hour window").
858const xaiFreeUsageExhaustedCooldown = 24 * time.Hour858const xaiFreeUsageExhaustedCooldown = 24 * time.Hour
859 859
860// xaiStatusErr wraps upstream error bodies so free-tier exhaustion860// xaiStatusErr normalizes upstream xAI error bodies for conductor behavior:
861// (subscription:free-usage-exhausted) carries a 24h RetryAfter hint for861// - credential invalidation (403 bad-credentials) is remapped to 401 so the
862// auth cooldown / account rotation. Generic 429s stay without an explicit862// existing OAuth refresh-once-and-retry path runs instead of payment cooldown
863// retry hint so conductor backoff still applies.863// - free-tier exhaustion (subscription:free-usage-exhausted) carries a 24h
864// RetryAfter hint for auth cooldown / account rotation
865//
866// Generic 429s stay without an explicit retry hint so conductor backoff still applies.
864func xaiStatusErr(code int, body []byte) statusErr {867func xaiStatusErr(code int, body []byte) statusErr {
865 err := statusErr{code: code, msg: string(body)}868 err := statusErr{code: code, msg: string(body)}
866 if code != http.StatusTooManyRequests || len(body) == 0 {869 if len(body) == 0 {
870 return err
871 }
872 if code == http.StatusForbidden && isXAIBadCredentialsBody(body) {
873 // Upstream returns 403 for invalidated OAuth access tokens. Map to 401 so
874 // tryRefreshAfterUnauthorized / MarkResult unauthorized handling applies.
875 err.code = http.StatusUnauthorized
876 return err
877 }
878 if code != http.StatusTooManyRequests {
867 return err879 return err
868 }880 }
869 codeStr := strings.ToLower(gjson.GetBytes(body, "code").String())881 codeStr := strings.ToLower(gjson.GetBytes(body, "code").String())
@@ -879,3 +891,24 @@ func xaiStatusErr(code int, body []byte) statusErr { · 9 unmodified lines
879 }891 }
880 return err892 return err
881}893}
894
895// isXAIBadCredentialsBody reports whether an xAI error body indicates an
896// invalidated/unusable OAuth access token rather than a generic permission or
897// payment failure. HTTP and websocket payloads both use this helper, so nested
898// error.code / error.message shapes are checked as well as flat bodies.
899func isXAIBadCredentialsBody(body []byte) bool {
900 for _, path := range []string{"code", "error.code", "body.error.code"} {
901 if strings.Contains(strings.ToLower(gjson.GetBytes(body, path).String()), "bad-credentials") {
902 return true
903 }
904 }
905 for _, path := range []string{"error", "error.message", "message", "body.error", "body.error.message"} {
906 msg := strings.ToLower(gjson.GetBytes(body, path).String())
907 if strings.Contains(msg, "access token could not be validated") {
908 return true
909 }
910 }
911 raw := strings.ToLower(string(body))
912 return strings.Contains(raw, "bad-credentials") ||
913 strings.Contains(raw, "access token could not be validated")
914}
internal/runtime/executor/xai_status_err_test.go+52−0
@@ -2,6 +2,7 @@ package executor · 1 unmodified line
2 2
3import (3import (
4 "net/http"4 "net/http"
5 "strings"
5 "testing"6 "testing"
6 "time"7 "time"
7)8)
@@ -31,7 +32,58 @@ func TestXAIStatusErr_Generic429HasNoRetryAfter(t *testing.T) { · 23 unmodified lines
31func TestXAIStatusErr_Non429Unchanged(t *testing.T) {32func TestXAIStatusErr_Non429Unchanged(t *testing.T) {
32 body := []byte(`{"error":"nope"}`)33 body := []byte(`{"error":"nope"}`)
33 err := xaiStatusErr(http.StatusBadRequest, body)34 err := xaiStatusErr(http.StatusBadRequest, body)
35 if err.StatusCode() != http.StatusBadRequest {
36 t.Fatalf("status = %d, want 400", err.StatusCode())
37 }
34 if err.RetryAfter() != nil {38 if err.RetryAfter() != nil {
35 t.Fatalf("expected nil RetryAfter for 400, got %v", *err.RetryAfter())39 t.Fatalf("expected nil RetryAfter for 400, got %v", *err.RetryAfter())
36 }40 }
37}41}
42
43func TestXAIStatusErr_BadCredentials403RemapsToUnauthorized(t *testing.T) {
44 body := []byte(`{"code":"unauthenticated:bad-credentials","error":"The OAuth2 access token could not be validated."}`)
45 err := xaiStatusErr(http.StatusForbidden, body)
46 if err.StatusCode() != http.StatusUnauthorized {
47 t.Fatalf("status = %d, want 401", err.StatusCode())
48 }
49 if !strings.Contains(err.Error(), "bad-credentials") {
50 t.Fatalf("error body should be preserved, got %q", err.Error())
51 }
52 if err.RetryAfter() != nil {
53 t.Fatalf("expected nil RetryAfter for bad-credentials, got %v", *err.RetryAfter())
54 }
55}
56
57func TestXAIStatusErr_BadCredentialsByMessageOnly(t *testing.T) {
58 body := []byte(`{"error":"The OAuth2 access token could not be validated."}`)
59 err := xaiStatusErr(http.StatusForbidden, body)
60 if err.StatusCode() != http.StatusUnauthorized {
61 t.Fatalf("status = %d, want 401", err.StatusCode())
62 }
63}
64
65func TestXAIStatusErr_BadCredentialsNestedErrorCode(t *testing.T) {
66 body := []byte(`{"type":"error","status":403,"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`)
67 err := xaiStatusErr(http.StatusForbidden, body)
68 if err.StatusCode() != http.StatusUnauthorized {
69 t.Fatalf("status = %d, want 401", err.StatusCode())
70 }
71}
72
73func TestXAIStatusErr_Generic403Unchanged(t *testing.T) {
74 body := []byte(`{"code":"permission_denied","error":"model access is not allowed for this account"}`)
75 err := xaiStatusErr(http.StatusForbidden, body)
76 if err.StatusCode() != http.StatusForbidden {
77 t.Fatalf("status = %d, want 403", err.StatusCode())
78 }
79 if err.RetryAfter() != nil {
80 t.Fatalf("expected nil RetryAfter for generic 403, got %v", *err.RetryAfter())
81 }
82}
83
84func TestXAIStatusErr_EmptyBodyForbiddenUnchanged(t *testing.T) {
85 err := xaiStatusErr(http.StatusForbidden, nil)
86 if err.StatusCode() != http.StatusForbidden {
87 t.Fatalf("status = %d, want 403", err.StatusCode())
88 }
89}
internal/runtime/executor/xai_websockets_executor.go+3−0
@@ -982,6 +982,9 @@ func parseXAIWebsocketError(payload []byte) (error, bool) { · 981 unmodified lines
982 if wsErr, ok := parseCodexWebsocketError(payload); ok {982 if wsErr, ok := parseCodexWebsocketError(payload); ok {
983 if statusError, okStatus := wsErr.(statusErrWithHeaders); okStatus {983 if statusError, okStatus := wsErr.(statusErrWithHeaders); okStatus {
984 xaiError := xaiStatusErr(statusError.code, payload)984 xaiError := xaiStatusErr(statusError.code, payload)
985 // Apply normalized status (e.g. 403 bad-credentials -> 401) and any
986 // provider-specific retry hint while preserving websocket headers.
987 statusError.code = xaiError.code
985 if xaiError.retryAfter != nil {988 if xaiError.retryAfter != nil {
986 statusError.retryAfter = xaiError.retryAfter989 statusError.retryAfter = xaiError.retryAfter
987 }990 }
internal/runtime/executor/xai_websockets_executor_test.go+37−0
@@ -1600,6 +1600,43 @@ func TestParseXAIWebsocketErrorFreeUsageExhaustedSetsRetryAfter(t *testing.T) { · 1599 unmodified lines
1600 }1600 }
1601}1601}
1602 1602
1603func TestParseXAIWebsocketErrorBadCredentialsRemapsToUnauthorized(t *testing.T) {
1604 payload := []byte(`{"type":"error","status":403,"headers":{"x-request-id":"req-bad-credentials"},"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`)
1605 err, ok := parseXAIWebsocketError(payload)
1606 if !ok {
1607 t.Fatal("expected xAI websocket error")
1608 }
1609
1610 status, okStatus := err.(interface{ StatusCode() int })
1611 if !okStatus || status.StatusCode() != http.StatusUnauthorized {
1612 t.Fatalf("status = %#v, want 401", err)
1613 }
1614 headerSource, okHeaders := err.(interface{ Headers() http.Header })
1615 if !okHeaders {
1616 t.Fatalf("expected websocket error to preserve headers, got %#v", err)
1617 }
1618 if got := headerSource.Headers().Get("x-request-id"); got != "req-bad-credentials" {
1619 t.Fatalf("x-request-id = %q, want req-bad-credentials", got)
1620 }
1621 parsed := gjson.Parse(err.Error())
1622 if got := parsed.Get("error.code").String(); got != "unauthenticated:bad-credentials" {
1623 t.Fatalf("error code = %q, want unauthenticated:bad-credentials; payload=%s", got, err)
1624 }
1625}
1626
1627func TestParseXAIWebsocketBareErrorBadCredentialsRemapsToUnauthorized(t *testing.T) {
1628 payload := []byte(`{"status":403,"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`)
1629 err, ok := parseXAIWebsocketError(payload)
1630 if !ok {
1631 t.Fatal("expected bare xAI websocket error")
1632 }
1633
1634 status, okStatus := err.(interface{ StatusCode() int })
1635 if !okStatus || status.StatusCode() != http.StatusUnauthorized {
1636 t.Fatalf("status = %#v, want 401", err)
1637 }
1638}
1639
1603func TestParseXAIWebsocketBareErrorFreeUsageExhaustedSetsRetryAfter(t *testing.T) {1640func TestParseXAIWebsocketBareErrorFreeUsageExhaustedSetsRetryAfter(t *testing.T) {
1604 payload := []byte(`{"status":429,"error":{"code":"subscription:free-usage-exhausted","message":"You've used all the included free usage for now."}}`)1641 payload := []byte(`{"status":429,"error":{"code":"subscription:free-usage-exhausted","message":"You've used all the included free usage for now."}}`)
1605 err, ok := parseXAIWebsocketError(payload)1642 err, ok := parseXAIWebsocketError(payload)

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t