- normalize xAI `403` bad-credentials responses to `401` before conductor retry handling - preserve websocket error headers while applying the normalized status/retry hints - share bad-credentials detection across flat and nested payload shapes for HTTP/websocket paths
Unified · Split
| @@ -857,13 +857,25 @@ func xaiPatchCompletedOutput(eventData []byte, outputItemsByIndex map[int64][]by · 856 unmodified lines | |||
| 857 | // cli-chat-proxy ("Usage resets over a rolling 24-hour window"). | 857 | // cli-chat-proxy ("Usage resets over a rolling 24-hour window"). |
| 858 | const xaiFreeUsageExhaustedCooldown = 24 * time.Hour | 858 | const xaiFreeUsageExhaustedCooldown = 24 * time.Hour |
| 859 | 859 | ||
| 860 | // xaiStatusErr wraps upstream error bodies so free-tier exhaustion | 860 | // xaiStatusErr normalizes upstream xAI error bodies for conductor behavior: |
| 861 | // (subscription:free-usage-exhausted) carries a 24h RetryAfter hint for | 861 | // - credential invalidation (403 bad-credentials) is remapped to 401 so the |
| 862 | // auth cooldown / account rotation. Generic 429s stay without an explicit | 862 | // existing OAuth refresh-once-and-retry path runs instead of payment cooldown |
| 863 | // retry hint so conductor backoff still applies. | 863 | // - free-tier exhaustion (subscription:free-usage-exhausted) carries a 24h |
| 864 | // RetryAfter hint for auth cooldown / account rotation | ||
| 865 | // | ||
| 866 | // Generic 429s stay without an explicit retry hint so conductor backoff still applies. | ||
| 864 | func xaiStatusErr(code int, body []byte) statusErr { | 867 | func xaiStatusErr(code int, body []byte) statusErr { |
| 865 | err := statusErr{code: code, msg: string(body)} | 868 | err := statusErr{code: code, msg: string(body)} |
| 866 | if code != http.StatusTooManyRequests || len(body) == 0 { | 869 | if len(body) == 0 { |
| 870 | return err | ||
| 871 | } | ||
| 872 | if code == http.StatusForbidden && isXAIBadCredentialsBody(body) { | ||
| 873 | // Upstream returns 403 for invalidated OAuth access tokens. Map to 401 so | ||
| 874 | // tryRefreshAfterUnauthorized / MarkResult unauthorized handling applies. | ||
| 875 | err.code = http.StatusUnauthorized | ||
| 876 | return err | ||
| 877 | } | ||
| 878 | if code != http.StatusTooManyRequests { | ||
| 867 | return err | 879 | return err |
| 868 | } | 880 | } |
| 869 | codeStr := strings.ToLower(gjson.GetBytes(body, "code").String()) | 881 | codeStr := strings.ToLower(gjson.GetBytes(body, "code").String()) |
| @@ -879,3 +891,24 @@ func xaiStatusErr(code int, body []byte) statusErr { · 9 unmodified lines | |||
| 879 | } | 891 | } |
| 880 | return err | 892 | return err |
| 881 | } | 893 | } |
| 894 | |||
| 895 | // isXAIBadCredentialsBody reports whether an xAI error body indicates an | ||
| 896 | // invalidated/unusable OAuth access token rather than a generic permission or | ||
| 897 | // payment failure. HTTP and websocket payloads both use this helper, so nested | ||
| 898 | // error.code / error.message shapes are checked as well as flat bodies. | ||
| 899 | func isXAIBadCredentialsBody(body []byte) bool { | ||
| 900 | for _, path := range []string{"code", "error.code", "body.error.code"} { | ||
| 901 | if strings.Contains(strings.ToLower(gjson.GetBytes(body, path).String()), "bad-credentials") { | ||
| 902 | return true | ||
| 903 | } | ||
| 904 | } | ||
| 905 | for _, path := range []string{"error", "error.message", "message", "body.error", "body.error.message"} { | ||
| 906 | msg := strings.ToLower(gjson.GetBytes(body, path).String()) | ||
| 907 | if strings.Contains(msg, "access token could not be validated") { | ||
| 908 | return true | ||
| 909 | } | ||
| 910 | } | ||
| 911 | raw := strings.ToLower(string(body)) | ||
| 912 | return strings.Contains(raw, "bad-credentials") || | ||
| 913 | strings.Contains(raw, "access token could not be validated") | ||
| 914 | } | ||
| @@ -2,6 +2,7 @@ package executor · 1 unmodified line | |||
| 2 | 2 | ||
| 3 | import ( | 3 | import ( |
| 4 | "net/http" | 4 | "net/http" |
| 5 | "strings" | ||
| 5 | "testing" | 6 | "testing" |
| 6 | "time" | 7 | "time" |
| 7 | ) | 8 | ) |
| @@ -31,7 +32,58 @@ func TestXAIStatusErr_Generic429HasNoRetryAfter(t *testing.T) { · 23 unmodified lines | |||
| 31 | func TestXAIStatusErr_Non429Unchanged(t *testing.T) { | 32 | func TestXAIStatusErr_Non429Unchanged(t *testing.T) { |
| 32 | body := []byte(`{"error":"nope"}`) | 33 | body := []byte(`{"error":"nope"}`) |
| 33 | err := xaiStatusErr(http.StatusBadRequest, body) | 34 | err := xaiStatusErr(http.StatusBadRequest, body) |
| 35 | if err.StatusCode() != http.StatusBadRequest { | ||
| 36 | t.Fatalf("status = %d, want 400", err.StatusCode()) | ||
| 37 | } | ||
| 34 | if err.RetryAfter() != nil { | 38 | if err.RetryAfter() != nil { |
| 35 | t.Fatalf("expected nil RetryAfter for 400, got %v", *err.RetryAfter()) | 39 | t.Fatalf("expected nil RetryAfter for 400, got %v", *err.RetryAfter()) |
| 36 | } | 40 | } |
| 37 | } | 41 | } |
| 42 | |||
| 43 | func TestXAIStatusErr_BadCredentials403RemapsToUnauthorized(t *testing.T) { | ||
| 44 | body := []byte(`{"code":"unauthenticated:bad-credentials","error":"The OAuth2 access token could not be validated."}`) | ||
| 45 | err := xaiStatusErr(http.StatusForbidden, body) | ||
| 46 | if err.StatusCode() != http.StatusUnauthorized { | ||
| 47 | t.Fatalf("status = %d, want 401", err.StatusCode()) | ||
| 48 | } | ||
| 49 | if !strings.Contains(err.Error(), "bad-credentials") { | ||
| 50 | t.Fatalf("error body should be preserved, got %q", err.Error()) | ||
| 51 | } | ||
| 52 | if err.RetryAfter() != nil { | ||
| 53 | t.Fatalf("expected nil RetryAfter for bad-credentials, got %v", *err.RetryAfter()) | ||
| 54 | } | ||
| 55 | } | ||
| 56 | |||
| 57 | func TestXAIStatusErr_BadCredentialsByMessageOnly(t *testing.T) { | ||
| 58 | body := []byte(`{"error":"The OAuth2 access token could not be validated."}`) | ||
| 59 | err := xaiStatusErr(http.StatusForbidden, body) | ||
| 60 | if err.StatusCode() != http.StatusUnauthorized { | ||
| 61 | t.Fatalf("status = %d, want 401", err.StatusCode()) | ||
| 62 | } | ||
| 63 | } | ||
| 64 | |||
| 65 | func TestXAIStatusErr_BadCredentialsNestedErrorCode(t *testing.T) { | ||
| 66 | body := []byte(`{"type":"error","status":403,"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`) | ||
| 67 | err := xaiStatusErr(http.StatusForbidden, body) | ||
| 68 | if err.StatusCode() != http.StatusUnauthorized { | ||
| 69 | t.Fatalf("status = %d, want 401", err.StatusCode()) | ||
| 70 | } | ||
| 71 | } | ||
| 72 | |||
| 73 | func TestXAIStatusErr_Generic403Unchanged(t *testing.T) { | ||
| 74 | body := []byte(`{"code":"permission_denied","error":"model access is not allowed for this account"}`) | ||
| 75 | err := xaiStatusErr(http.StatusForbidden, body) | ||
| 76 | if err.StatusCode() != http.StatusForbidden { | ||
| 77 | t.Fatalf("status = %d, want 403", err.StatusCode()) | ||
| 78 | } | ||
| 79 | if err.RetryAfter() != nil { | ||
| 80 | t.Fatalf("expected nil RetryAfter for generic 403, got %v", *err.RetryAfter()) | ||
| 81 | } | ||
| 82 | } | ||
| 83 | |||
| 84 | func TestXAIStatusErr_EmptyBodyForbiddenUnchanged(t *testing.T) { | ||
| 85 | err := xaiStatusErr(http.StatusForbidden, nil) | ||
| 86 | if err.StatusCode() != http.StatusForbidden { | ||
| 87 | t.Fatalf("status = %d, want 403", err.StatusCode()) | ||
| 88 | } | ||
| 89 | } | ||
| @@ -982,6 +982,9 @@ func parseXAIWebsocketError(payload []byte) (error, bool) { · 981 unmodified lines | |||
| 982 | if wsErr, ok := parseCodexWebsocketError(payload); ok { | 982 | if wsErr, ok := parseCodexWebsocketError(payload); ok { |
| 983 | if statusError, okStatus := wsErr.(statusErrWithHeaders); okStatus { | 983 | if statusError, okStatus := wsErr.(statusErrWithHeaders); okStatus { |
| 984 | xaiError := xaiStatusErr(statusError.code, payload) | 984 | xaiError := xaiStatusErr(statusError.code, payload) |
| 985 | // Apply normalized status (e.g. 403 bad-credentials -> 401) and any | ||
| 986 | // provider-specific retry hint while preserving websocket headers. | ||
| 987 | statusError.code = xaiError.code | ||
| 985 | if xaiError.retryAfter != nil { | 988 | if xaiError.retryAfter != nil { |
| 986 | statusError.retryAfter = xaiError.retryAfter | 989 | statusError.retryAfter = xaiError.retryAfter |
| 987 | } | 990 | } |
| @@ -1600,6 +1600,43 @@ func TestParseXAIWebsocketErrorFreeUsageExhaustedSetsRetryAfter(t *testing.T) { · 1599 unmodified lines | |||
| 1600 | } | 1600 | } |
| 1601 | } | 1601 | } |
| 1602 | 1602 | ||
| 1603 | func TestParseXAIWebsocketErrorBadCredentialsRemapsToUnauthorized(t *testing.T) { | ||
| 1604 | payload := []byte(`{"type":"error","status":403,"headers":{"x-request-id":"req-bad-credentials"},"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`) | ||
| 1605 | err, ok := parseXAIWebsocketError(payload) | ||
| 1606 | if !ok { | ||
| 1607 | t.Fatal("expected xAI websocket error") | ||
| 1608 | } | ||
| 1609 | |||
| 1610 | status, okStatus := err.(interface{ StatusCode() int }) | ||
| 1611 | if !okStatus || status.StatusCode() != http.StatusUnauthorized { | ||
| 1612 | t.Fatalf("status = %#v, want 401", err) | ||
| 1613 | } | ||
| 1614 | headerSource, okHeaders := err.(interface{ Headers() http.Header }) | ||
| 1615 | if !okHeaders { | ||
| 1616 | t.Fatalf("expected websocket error to preserve headers, got %#v", err) | ||
| 1617 | } | ||
| 1618 | if got := headerSource.Headers().Get("x-request-id"); got != "req-bad-credentials" { | ||
| 1619 | t.Fatalf("x-request-id = %q, want req-bad-credentials", got) | ||
| 1620 | } | ||
| 1621 | parsed := gjson.Parse(err.Error()) | ||
| 1622 | if got := parsed.Get("error.code").String(); got != "unauthenticated:bad-credentials" { | ||
| 1623 | t.Fatalf("error code = %q, want unauthenticated:bad-credentials; payload=%s", got, err) | ||
| 1624 | } | ||
| 1625 | } | ||
| 1626 | |||
| 1627 | func TestParseXAIWebsocketBareErrorBadCredentialsRemapsToUnauthorized(t *testing.T) { | ||
| 1628 | payload := []byte(`{"status":403,"error":{"code":"unauthenticated:bad-credentials","message":"The OAuth2 access token could not be validated."}}`) | ||
| 1629 | err, ok := parseXAIWebsocketError(payload) | ||
| 1630 | if !ok { | ||
| 1631 | t.Fatal("expected bare xAI websocket error") | ||
| 1632 | } | ||
| 1633 | |||
| 1634 | status, okStatus := err.(interface{ StatusCode() int }) | ||
| 1635 | if !okStatus || status.StatusCode() != http.StatusUnauthorized { | ||
| 1636 | t.Fatalf("status = %#v, want 401", err) | ||
| 1637 | } | ||
| 1638 | } | ||
| 1639 | |||
| 1603 | func TestParseXAIWebsocketBareErrorFreeUsageExhaustedSetsRetryAfter(t *testing.T) { | 1640 | func TestParseXAIWebsocketBareErrorFreeUsageExhaustedSetsRetryAfter(t *testing.T) { |
| 1604 | payload := []byte(`{"status":429,"error":{"code":"subscription:free-usage-exhausted","message":"You've used all the included free usage for now."}}`) | 1641 | payload := []byte(`{"status":429,"error":{"code":"subscription:free-usage-exhausted","message":"You've used all the included free usage for now."}}`) |
| 1605 | err, ok := parseXAIWebsocketError(payload) | 1642 | err, ok := parseXAIWebsocketError(payload) |