- normalize xAI `403` bad-credentials responses to `401` before conductor retry handling - preserve websocket error headers while applying the normalized status/retry hints - share bad-credentials detection across flat and nested payload shapes for HTTP/websocket paths
Unified · Split
| @@ -857,13 +857,25 @@ func xaiPatchCompletedOutput(eventData []byte, outputItemsByIndex map[int64][]by · 856 unmodified lines | ||
| 857 | 857 | // cli-chat-proxy ("Usage resets over a rolling 24-hour window"). |
| 858 | 858 | const xaiFreeUsageExhaustedCooldown = 24 * time.Hour |
| 859 | 859 | |
| 860 | −// xaiStatusErr wraps upstream error bodies so free-tier exhaustion | |
| 861 | −// (subscription:free-usage-exhausted) carries a 24h RetryAfter hint for | |
| 862 | −// auth cooldown / account rotation. Generic 429s stay without an explicit | |
| 863 | −// retry hint so conductor backoff still applies. | |
| 860 | +// xaiStatusErr normalizes upstream xAI error bodies for conductor behavior: | |
| 861 | +// - credential invalidation (403 bad-credentials) is remapped to 401 so the | |
| 862 | +// existing OAuth refresh-once-and-retry path runs instead of payment cooldown | |
| 863 | +// - free-tier exhaustion (subscription:free-usage-exhausted) carries a 24h | |
| 864 | +// RetryAfter hint for auth cooldown / account rotation | |
| 865 | +// | |
| 866 | +// Generic 429s stay without an explicit retry hint so conductor backoff still applies. | |
| 864 | 867 | func xaiStatusErr(code int, body []byte) statusErr { |
| 865 | 868 | err := statusErr{code: code, msg: string(body)} |
| 866 | − if code != http.StatusTooManyRequests || len(body) == 0 { | |
| 869 | + if len(body) == 0 { | |
| 870 | + return err | |
| 871 | + } | |
| 872 | + if code == http.StatusForbidden && isXAIBadCredentialsBody(body) { | |
| 873 | + // Upstream returns 403 for invalidated OAuth access tokens. Map to 401 so | |
| 874 | + // tryRefreshAfterUnauthorized / MarkResult unauthorized handling applies. | |
| 875 | + err.code = http.StatusUnauthorized | |
| 876 | + return err | |
| 877 | + } | |
| 878 | + if code != http.StatusTooManyRequests { | |
| 867 | 879 | return err |
| 868 | 880 | } |
| 869 | 881 | codeStr := strings.ToLower(gjson.GetBytes(body, "code").String()) |
| @@ -879,3 +891,24 @@ func xaiStatusErr(code int, body []byte) statusErr { · 9 unmodified lines | ||
| 879 | 891 | } |
| 880 | 892 | return err |
| 881 | 893 | } |
| 894 | + | |
| 895 | +// isXAIBadCredentialsBody reports whether an xAI error body indicates an | |
| 896 | +// invalidated/unusable OAuth access token rather than a generic permission or | |
| 897 | +// payment failure. HTTP and websocket payloads both use this helper, so nested | |
| 898 | +// error.code / error.message shapes are checked as well as flat bodies. | |
| 899 | +func isXAIBadCredentialsBody(body []byte) bool { | |
| 900 | + for _, path := range []string{"code", "error.code", "body.error.code"} { | |
| 901 | + if strings.Contains(strings.ToLower(gjson.GetBytes(body, path).String()), "bad-credentials") { | |
| 902 | + return true | |
| 903 | + } | |
| 904 | + } | |
| 905 | + for _, path := range []string{"error", "error.message", "message", "body.error", "body.error.message"} { | |
| 906 | + msg := strings.ToLower(gjson.GetBytes(body, path).String()) | |
| 907 | + if strings.Contains(msg, "access token could not be validated") { | |
| 908 | + return true | |
| 909 | + } | |
| 910 | + } | |
| 911 | + raw := strings.ToLower(string(body)) | |
| 912 | + return strings.Contains(raw, "bad-credentials") || | |
| 913 | + strings.Contains(raw, "access token could not be validated") | |
| 914 | +} | |
| @@ -2,6 +2,7 @@ package executor · 1 unmodified line | ||
| 2 | 2 | |
| 3 | 3 | import ( |
| 4 | 4 | "net/http" |
| 5 | + "strings" | |
| 5 | 6 | "testing" |
| 6 | 7 | "time" |
| 7 | 8 | ) |
| @@ -31,7 +32,58 @@ func TestXAIStatusErr_Generic429HasNoRetryAfter(t *testing.T) { · 23 unmodified lines | ||
| 31 | 32 | func TestXAIStatusErr_Non429Unchanged(t *testing.T) { |
| 32 | 33 | body := []byte(`{"error":"nope"}`) |
| 33 | 34 | err := xaiStatusErr(http.StatusBadRequest, body) |
| 35 | + if err.StatusCode() != http.StatusBadRequest { | |
| 36 | + t.Fatalf("status = %d, want 400", err.StatusCode()) | |
| 37 | + } | |
| 34 | 38 | if err.RetryAfter() != nil { |
| 35 | 39 | t.Fatalf("expected nil RetryAfter for 400, got %v", *err.RetryAfter()) |
| 36 | 40 | } |
| 37 | 41 | } |
| 42 | + | |
| 43 | +func TestXAIStatusErr_BadCredentials403RemapsToUnauthorized(t *testing.T) { | |
| 44 | + body := []byte(`{"code":"unauthenticated:bad-credentials","error":"The OAuth2 access token could not be validated."}`) | |
| 45 | + err := xaiStatusErr(http.StatusForbidden, body) | |
| 46 | + if err.StatusCode() != http.StatusUnauthorized { | |
| 47 | + t.Fatalf("status = %d, want 401", err.StatusCode()) | |
| 48 | + } | |
| 49 | + if !strings.Contains(err.Error(), "bad-credentials") { | |
| 50 | + t.Fatalf("error body should be preserved, got %q", err.Error()) | |
| 51 | + } | |
| 52 | + if err.RetryAfter() != nil { | |
| 53 | + t.Fatalf("expected nil RetryAfter for bad-credentials, got %v", *err.RetryAfter()) | |
| 54 | + } | |
| 55 | +} | |
| 56 | + | |
| 57 | +func TestXAIStatusErr_BadCredentialsByMessageOnly(t *testing.T) { | |
| 58 | + body := []byte(`{"error":"The OAuth2 access token could not be validated."}`) | |
| 59 | + err := xaiStatusErr(http.StatusForbidden, body) | |
| 60 | + if err.StatusCode() != http.StatusUnauthorized { | |
| 61 | + t.Fatalf("status = %d, want 401", err.StatusCode()) | |
| 62 | + } | |
| 63 | +} | |
| 64 | + | |
| 65 | +func TestXAIStatusErr_BadCredentialsNestedErrorCode(t *testing.T) { | |
| 66 | + body := []byte(`{"type":"error","status":403,"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`) | |
| 67 | + err := xaiStatusErr(http.StatusForbidden, body) | |
| 68 | + if err.StatusCode() != http.StatusUnauthorized { | |
| 69 | + t.Fatalf("status = %d, want 401", err.StatusCode()) | |
| 70 | + } | |
| 71 | +} | |
| 72 | + | |
| 73 | +func TestXAIStatusErr_Generic403Unchanged(t *testing.T) { | |
| 74 | + body := []byte(`{"code":"permission_denied","error":"model access is not allowed for this account"}`) | |
| 75 | + err := xaiStatusErr(http.StatusForbidden, body) | |
| 76 | + if err.StatusCode() != http.StatusForbidden { | |
| 77 | + t.Fatalf("status = %d, want 403", err.StatusCode()) | |
| 78 | + } | |
| 79 | + if err.RetryAfter() != nil { | |
| 80 | + t.Fatalf("expected nil RetryAfter for generic 403, got %v", *err.RetryAfter()) | |
| 81 | + } | |
| 82 | +} | |
| 83 | + | |
| 84 | +func TestXAIStatusErr_EmptyBodyForbiddenUnchanged(t *testing.T) { | |
| 85 | + err := xaiStatusErr(http.StatusForbidden, nil) | |
| 86 | + if err.StatusCode() != http.StatusForbidden { | |
| 87 | + t.Fatalf("status = %d, want 403", err.StatusCode()) | |
| 88 | + } | |
| 89 | +} | |
| @@ -982,6 +982,9 @@ func parseXAIWebsocketError(payload []byte) (error, bool) { · 981 unmodified lines | ||
| 982 | 982 | if wsErr, ok := parseCodexWebsocketError(payload); ok { |
| 983 | 983 | if statusError, okStatus := wsErr.(statusErrWithHeaders); okStatus { |
| 984 | 984 | xaiError := xaiStatusErr(statusError.code, payload) |
| 985 | + // Apply normalized status (e.g. 403 bad-credentials -> 401) and any | |
| 986 | + // provider-specific retry hint while preserving websocket headers. | |
| 987 | + statusError.code = xaiError.code | |
| 985 | 988 | if xaiError.retryAfter != nil { |
| 986 | 989 | statusError.retryAfter = xaiError.retryAfter |
| 987 | 990 | } |
| @@ -1600,6 +1600,43 @@ func TestParseXAIWebsocketErrorFreeUsageExhaustedSetsRetryAfter(t *testing.T) { · 1599 unmodified lines | ||
| 1600 | 1600 | } |
| 1601 | 1601 | } |
| 1602 | 1602 | |
| 1603 | +func TestParseXAIWebsocketErrorBadCredentialsRemapsToUnauthorized(t *testing.T) { | |
| 1604 | + payload := []byte(`{"type":"error","status":403,"headers":{"x-request-id":"req-bad-credentials"},"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`) | |
| 1605 | + err, ok := parseXAIWebsocketError(payload) | |
| 1606 | + if !ok { | |
| 1607 | + t.Fatal("expected xAI websocket error") | |
| 1608 | + } | |
| 1609 | + | |
| 1610 | + status, okStatus := err.(interface{ StatusCode() int }) | |
| 1611 | + if !okStatus || status.StatusCode() != http.StatusUnauthorized { | |
| 1612 | + t.Fatalf("status = %#v, want 401", err) | |
| 1613 | + } | |
| 1614 | + headerSource, okHeaders := err.(interface{ Headers() http.Header }) | |
| 1615 | + if !okHeaders { | |
| 1616 | + t.Fatalf("expected websocket error to preserve headers, got %#v", err) | |
| 1617 | + } | |
| 1618 | + if got := headerSource.Headers().Get("x-request-id"); got != "req-bad-credentials" { | |
| 1619 | + t.Fatalf("x-request-id = %q, want req-bad-credentials", got) | |
| 1620 | + } | |
| 1621 | + parsed := gjson.Parse(err.Error()) | |
| 1622 | + if got := parsed.Get("error.code").String(); got != "unauthenticated:bad-credentials" { | |
| 1623 | + t.Fatalf("error code = %q, want unauthenticated:bad-credentials; payload=%s", got, err) | |
| 1624 | + } | |
| 1625 | +} | |
| 1626 | + | |
| 1627 | +func TestParseXAIWebsocketBareErrorBadCredentialsRemapsToUnauthorized(t *testing.T) { | |
| 1628 | + payload := []byte(`{"status":403,"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`) | |
| 1629 | + err, ok := parseXAIWebsocketError(payload) | |
| 1630 | + if !ok { | |
| 1631 | + t.Fatal("expected bare xAI websocket error") | |
| 1632 | + } | |
| 1633 | + | |
| 1634 | + status, okStatus := err.(interface{ StatusCode() int }) | |
| 1635 | + if !okStatus || status.StatusCode() != http.StatusUnauthorized { | |
| 1636 | + t.Fatalf("status = %#v, want 401", err) | |
| 1637 | + } | |
| 1638 | +} | |
| 1639 | + | |
| 1603 | 1640 | func TestParseXAIWebsocketBareErrorFreeUsageExhaustedSetsRetryAfter(t *testing.T) { |
| 1604 | 1641 | payload := []byte(`{"status":429,"error":{"code":"subscription:free-usage-exhausted","message":"You've used all the included free usage for now."}}`) |
| 1605 | 1642 | err, ok := parseXAIWebsocketError(payload) |