A vault write failure while persisting a refreshed OAuth token lost the rotated refresh token: the grant had already consumed the single-use stored token at the authorization server, so once the vault recovered the next refresh replayed the revoked token, got invalid_grant, and the connection demanded a re-auth over what was only a storage blip. Before consuming the refresh token, rewrite the stored value in place as a writability probe. A store outage now fails the resolve before the grant, leaving the stored token valid so the connection recovers with the store.
Unified · Split
| @@ -7,30 +7,31 @@ · 6 unmodified lines | |||
| 7 | // 1. An OAuth connection's access token expires (per-client TTL of 1s via | 7 | // 1. An OAuth connection's access token expires (per-client TTL of 1s via |
| 8 | // the emulator's DCR `access_token_ttl_seconds` extension, well inside | 8 | // the emulator's DCR `access_token_ttl_seconds` extension, well inside |
| 9 | // the 60s refresh skew). | 9 | // the 60s refresh skew). |
| 10 | // 2. A health check resolves the connection → triggers the refresh-token | 10 | // 2. A health check resolves the connection → the refresh path runs. Vault |
| 11 | // grant. The grant SUCCEEDS and the authorization server ROTATES the | 11 | // writes go through read → `PUT /vault/v1/kv/:id`; the PUT fails (in |
| 12 | // refresh token (single use — the emulator mirrors AuthKit). | 12 | // prod: 409 version conflict, an OAuth-shaped 400, or an HTML error |
| 13 | // 3. Persisting the new tokens does read → `PUT /vault/v1/kv/:id`. The PUT | 13 | // page; here: the armed 400 with `error`/`error_description`, the exact |
| 14 | // fails (in prod: 409 version conflict, an OAuth-shaped 400, or an HTML | 14 | // shape the WorkOS SDK maps to OauthException — Sentry issue -53). |
| 15 | // error page; here: the armed 400 with `error`/`error_description`, the | 15 | // 3. The write failure surfaces as `StorageError: WorkOS Vault secret write |
| 16 | // exact shape the WorkOS SDK maps to OauthException — Sentry issue -53). | ||
| 17 | // 4. The write failure surfaces as `StorageError: WorkOS Vault secret write | ||
| 18 | // failed` → the health endpoint answers a typed InternalError (the 500 | 16 | // failed` → the health endpoint answers a typed InternalError (the 500 |
| 19 | // Sentry records). | 17 | // Sentry records). Crucially it must fail BEFORE the refresh-token grant |
| 18 | // (the AS rotates the single-use token; consuming it with an unwritable | ||
| 19 | // store loses the rotated copy forever). | ||
| 20 | // | 20 | // |
| 21 | // The second scenario pins the DAMAGE, not just the surface error: the vault | 21 | // The second scenario pins the DAMAGE the surface error used to hide: when |
| 22 | // write failed AFTER the refresh token was consumed and rotated, so the new | 22 | // the vault write failed AFTER the refresh token was consumed and rotated, |
| 23 | // refresh token is lost and the stored one is revoked. Once the vault | 23 | // the rotated token was lost and the stored one already revoked — the next |
| 24 | // recovers, the connection must recover too — today it cannot (the next | 24 | // refresh got invalid_grant and the connection demanded a re-auth over a |
| 25 | // refresh gets invalid_grant → the connection reads "expired" and demands a | 25 | // storage blip. The fix gates the grant on a proof-of-writability rewrite of |
| 26 | // re-auth). Red until the refresh/persist ordering is made crash-safe. | 26 | // the stored refresh token, so a vault outage fails BEFORE the single-use |
| 27 | // token is spent and the connection recovers with the vault. | ||
| 27 | import { randomBytes } from "node:crypto"; | 28 | import { randomBytes } from "node:crypto"; |
| 28 | 29 | ||
| 29 | import { expect } from "@effect/vitest"; | 30 | import { expect } from "@effect/vitest"; |
| 30 | import { Effect } from "effect"; | 31 | import { Effect } from "effect"; |
| 31 | import type { HttpApiClient } from "effect/unstable/httpapi"; | 32 | import type { HttpApiClient } from "effect/unstable/httpapi"; |
| 32 | import { composePluginApi } from "@executor-js/api/server"; | 33 | import { composePluginApi } from "@executor-js/api/server"; |
| 33 | import { connectEmulator, type EmulatorClient } from "@executor-js/emulate"; | 34 | import { connectEmulator } from "@executor-js/emulate"; |
| 34 | import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; | 35 | import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; |
| 35 | import { | 36 | import { |
| 36 | AuthTemplateSlug, | 37 | AuthTemplateSlug, |
| @@ -243,13 +244,15 @@ scenario( · 206 unmodified lines | |||
| 243 | ).toBe("InternalError"); | 244 | ).toBe("InternalError"); |
| 244 | 245 | ||
| 245 | // The proof this is the production mechanism and not an incidental | 246 | // The proof this is the production mechanism and not an incidental |
| 246 | // failure: the product performed the refresh grant and then hit the | 247 | // failure: the product hit the injected fault on the vault update leg. |
| 247 | // injected fault on the vault update. | ||
| 248 | const ledger = yield* Effect.promise(() => workos.ledger.list(200)); | 248 | const ledger = yield* Effect.promise(() => workos.ledger.list(200)); |
| 249 | const faultedPut = ledger.find((entry) => entry.faulted === true && entry.method === "PUT"); | 249 | const faultedPut = ledger.find((entry) => entry.faulted === true && entry.method === "PUT"); |
| 250 | expect(faultedPut?.path, "the failure came from the injected vault-update fault").toMatch( | 250 | expect(faultedPut?.path, "the failure came from the injected vault-update fault").toMatch( |
| 251 | /\/vault\/v1\/kv\//, | 251 | /\/vault\/v1\/kv\//, |
| 252 | ); | 252 | ); |
| 253 | // The crash-safety contract: the write-gate fails BEFORE the refresh | ||
| 254 | // grant, so the single-use refresh token is never consumed while the | ||
| 255 | // store cannot persist its rotated successor. | ||
| 253 | const refreshGrant = ledger.find( | 256 | const refreshGrant = ledger.find( |
| 254 | (entry) => | 257 | (entry) => |
| 255 | entry.path.endsWith("/oauth2/token") && | 258 | entry.path.endsWith("/oauth2/token") && |
| @@ -257,8 +260,8 @@ scenario( · 1 unmodified line | |||
| 257 | ); | 260 | ); |
| 258 | expect( | 261 | expect( |
| 259 | refreshGrant, | 262 | refreshGrant, |
| 260 | "the vault write that failed was persisting a completed refresh grant", | 263 | "no refresh grant is issued while the vault cannot persist the rotated token", |
| 261 | ).toBeDefined(); | 264 | ).toBeUndefined(); |
| 262 | }), | 265 | }), |
| 263 | Effect.gen(function* () { | 266 | Effect.gen(function* () { |
| 264 | yield* Effect.promise(() => workos.faults.clear()); | 267 | yield* Effect.promise(() => workos.faults.clear()); |
| @@ -1570,6 +1570,17 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea · 1569 unmodified lines | |||
| 1570 | if (!refreshToken) { | 1570 | if (!refreshToken) { |
| 1571 | return yield* reauth("Stored refresh token could not be resolved."); | 1571 | return yield* reauth("Stored refresh token could not be resolved."); |
| 1572 | } | 1572 | } |
| 1573 | // Prove the credential store is WRITABLE before consuming the | ||
| 1574 | // single-use refresh token: the grant rotates it at the AS, so | ||
| 1575 | // a persist failure after the grant loses the only copy of the | ||
| 1576 | // rotated token and permanently invalidates the connection | ||
| 1577 | // (the next refresh replays the revoked token → invalid_grant | ||
| 1578 | // → forced re-auth). A no-op rewrite of the value just read | ||
| 1579 | // fails fast during a store outage, leaving the stored token | ||
| 1580 | // valid so the connection recovers when the store does. | ||
| 1581 | if (provider.set) { | ||
| 1582 | yield* provider.set(ProviderItemId.make(row.refresh_item_id), refreshToken); | ||
| 1583 | } | ||
| 1573 | return yield* refreshAccessToken({ | 1584 | return yield* refreshAccessToken({ |
| 1574 | tokenUrl, | 1585 | tokenUrl, |
| 1575 | clientId: String(clientRow.client_id), | 1586 | clientId: String(clientRow.client_id), |