Railway answers every scope-bearing refresh with `invalid_scope: refresh token missing requested scope` when its stored grant is narrower than the authorization it echoed back, so a connection whose refresh token was still live failed every call as `oauth_refresh_failed` and only a hand re-authorization recovered it. RFC 6749 §6 defines omitting `scope` as "the scope originally granted", so retry the grant once without it. Only `invalid_scope` qualifies: `invalid_grant` means the token is dead, and retrying that spends a rotating refresh token to learn nothing.
Showing the changes to e2e (1 of 5 files). Show all
Unified · Split
| @@ -0,0 +1,297 @@ | ||
| 1 | +// Cross-target: an authorization server that refuses a scope-bearing refresh | |
| 2 | +// grant must not turn a live refresh token into a dead connection. | |
| 3 | +// | |
| 4 | +// Railway answers any refresh grant carrying a `scope` parameter with | |
| 5 | +// `invalid_scope: refresh token missing requested scope` when the refresh | |
| 6 | +// token's own record is narrower than the authorization response it echoed | |
| 7 | +// back (issue #1969). Echoing the recorded grant is legal under RFC 6749 §6, | |
| 8 | +// and so is omitting `scope`, which the spec defines as "the scope originally | |
| 9 | +// granted". Before the fallback, the refusal reached the sandbox as | |
| 10 | +// `oauth_refresh_failed` with `retryable: false`, so a connection whose refresh | |
| 11 | +// token was perfectly alive stayed unusable until someone re-authorized it by | |
| 12 | +// hand — the reported symptom was a saved Railway connection failing every | |
| 13 | +// call. | |
| 14 | +// | |
| 15 | +// The journey: an OpenAPI integration completes a real authorization-code flow | |
| 16 | +// against a live test AS that mints instantly-expiring access tokens and holds | |
| 17 | +// a narrower grant on the refresh token than the authorization echoed. The | |
| 18 | +// first tool call must therefore refresh, the scope-bearing grant is refused, | |
| 19 | +// executor retries WITHOUT `scope`, and the call succeeds — proven from the | |
| 20 | +// AS's own request ledger, which records both the refused request and the | |
| 21 | +// accepted retry. | |
| 22 | +import { randomBytes } from "node:crypto"; | |
| 23 | +import { createServer } from "node:http"; | |
| 24 | + | |
| 25 | +import { expect } from "@effect/vitest"; | |
| 26 | +import { Effect } from "effect"; | |
| 27 | +import { composePluginApi } from "@executor-js/api/server"; | |
| 28 | +import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; | |
| 29 | +import { | |
| 30 | + AuthTemplateSlug, | |
| 31 | + ConnectionName, | |
| 32 | + IntegrationSlug, | |
| 33 | + OAuthClientSlug, | |
| 34 | +} from "@executor-js/sdk/shared"; | |
| 35 | +import { serveOAuthTestServer } from "@executor-js/sdk/testing"; | |
| 36 | + | |
| 37 | +import { scenario } from "../src/scenario"; | |
| 38 | +import { Api, Mcp, Target } from "../src/services"; | |
| 39 | + | |
| 40 | +const api = composePluginApi([openApiHttpPlugin()] as const); | |
| 41 | + | |
| 42 | +const unique = (prefix: string) => `${prefix}_${randomBytes(4).toString("hex")}`; | |
| 43 | + | |
| 44 | +/** Upstream on 127.0.0.1: `GET /issues` is 200 for any bearer and returns one | |
| 45 | + * issue, so "the call succeeded" is distinguishable from "the call returned | |
| 46 | + * an empty body". */ | |
| 47 | +const serveUpstream = () => | |
| 48 | + Effect.acquireRelease( | |
| 49 | + Effect.callback<{ readonly url: string; readonly close: () => void }>((resume) => { | |
| 50 | + const server = createServer((request, response) => { | |
| 51 | + if (request.method === "GET" && (request.url ?? "").startsWith("/issues")) { | |
| 52 | + response.writeHead(200, { "content-type": "application/json" }); | |
| 53 | + response.end(JSON.stringify({ issues: [{ id: "issue-1", title: "Scope drift" }] })); | |
| 54 | + return; | |
| 55 | + } | |
| 56 | + response.writeHead(404, { "content-type": "application/json" }); | |
| 57 | + response.end(JSON.stringify({ error: "not_found" })); | |
| 58 | + }); | |
| 59 | + server.listen(0, "127.0.0.1", () => { | |
| 60 | + const address = server.address(); | |
| 61 | + const port = typeof address === "object" && address ? address.port : 0; | |
| 62 | + resume( | |
| 63 | + Effect.succeed({ | |
| 64 | + url: `http://127.0.0.1:${port}`, | |
| 65 | + close: () => { | |
| 66 | + server.close(); | |
| 67 | + server.closeAllConnections(); | |
| 68 | + }, | |
| 69 | + }), | |
| 70 | + ); | |
| 71 | + }); | |
| 72 | + }), | |
| 73 | + (server) => Effect.sync(server.close), | |
| 74 | + ); | |
| 75 | + | |
| 76 | +const spec = ( | |
| 77 | + baseUrl: string, | |
| 78 | + oauth: { readonly authorizationEndpoint: string; readonly tokenEndpoint: string }, | |
| 79 | +): string => | |
| 80 | + JSON.stringify({ | |
| 81 | + openapi: "3.0.3", | |
| 82 | + info: { title: "Issues API", version: "1.0.0" }, | |
| 83 | + servers: [{ url: baseUrl }], | |
| 84 | + paths: { | |
| 85 | + "/issues": { | |
| 86 | + get: { | |
| 87 | + operationId: "listIssues", | |
| 88 | + summary: "List issues", | |
| 89 | + security: [{ oauth: ["issues.read"] }], | |
| 90 | + responses: { "200": { description: "issues" } }, | |
| 91 | + }, | |
| 92 | + }, | |
| 93 | + }, | |
| 94 | + components: { | |
| 95 | + securitySchemes: { | |
| 96 | + oauth: { | |
| 97 | + type: "oauth2", | |
| 98 | + flows: { | |
| 99 | + authorizationCode: { | |
| 100 | + authorizationUrl: oauth.authorizationEndpoint, | |
| 101 | + tokenUrl: oauth.tokenEndpoint, | |
| 102 | + scopes: { "issues.read": "Read issues", "issues.write": "Write issues" }, | |
| 103 | + }, | |
| 104 | + }, | |
| 105 | + }, | |
| 106 | + }, | |
| 107 | + }, | |
| 108 | + }); | |
| 109 | + | |
| 110 | +const invokeByAddressCode = (address: string, args: unknown) => ` | |
| 111 | +const segments = ${JSON.stringify(address)}.split(".").slice(1); | |
| 112 | +let node = tools; | |
| 113 | +for (const segment of segments) node = node[segment]; | |
| 114 | +const result = await node(${JSON.stringify(args)}); | |
| 115 | +return JSON.stringify(result); | |
| 116 | +`; | |
| 117 | + | |
| 118 | +type ToolEnvelope = { | |
| 119 | + readonly ok: boolean; | |
| 120 | + readonly data?: unknown; | |
| 121 | + readonly error?: { | |
| 122 | + readonly code?: string; | |
| 123 | + readonly message?: string; | |
| 124 | + }; | |
| 125 | +}; | |
| 126 | + | |
| 127 | +scenario( | |
| 128 | + "Auth failures · a refresh refused for its scope is retried without one, so a scope-drifted connection keeps working", | |
| 129 | + {}, | |
| 130 | + Effect.scoped( | |
| 131 | + Effect.gen(function* () { | |
| 132 | + const target = yield* Target; | |
| 133 | + const { client: makeClient } = yield* Api; | |
| 134 | + const mcp = yield* Mcp; | |
| 135 | + const identity = yield* target.newIdentity(); | |
| 136 | + const client = yield* makeClient(api, identity); | |
| 137 | + const upstream = yield* serveUpstream(); | |
| 138 | + // The AS grants both scopes at authorization and echoes them, but the | |
| 139 | + // refresh token it stores covers only `issues.read` — the divergence | |
| 140 | + // that makes the recorded grant look like a request for more access. | |
| 141 | + const oauth = yield* serveOAuthTestServer({ | |
| 142 | + scopes: ["issues.read", "issues.write"], | |
| 143 | + refreshGrantScopes: ["issues.read"], | |
| 144 | + tokenExpiresInSeconds: 0, | |
| 145 | + }); | |
| 146 | + const slug = unique("refreshscope"); | |
| 147 | + const clientSlug = OAuthClientSlug.make(unique("refreshscopec")); | |
| 148 | + | |
| 149 | + yield* Effect.ensuring( | |
| 150 | + Effect.gen(function* () { | |
| 151 | + yield* client.openapi.addSpec({ | |
| 152 | + payload: { | |
| 153 | + spec: { kind: "blob", value: spec(upstream.url, oauth) }, | |
| 154 | + slug, | |
| 155 | + baseUrl: upstream.url, | |
| 156 | + authenticationTemplate: [ | |
| 157 | + { | |
| 158 | + slug: "oauth", | |
| 159 | + kind: "oauth2", | |
| 160 | + authorizationUrl: oauth.authorizationEndpoint, | |
| 161 | + tokenUrl: oauth.tokenEndpoint, | |
| 162 | + scopes: ["issues.read", "issues.write"], | |
| 163 | + }, | |
| 164 | + ], | |
| 165 | + }, | |
| 166 | + }); | |
| 167 | + yield* client.oauth.createClient({ | |
| 168 | + payload: { | |
| 169 | + owner: "org", | |
| 170 | + slug: clientSlug, | |
| 171 | + grant: "authorization_code", | |
| 172 | + authorizationUrl: oauth.authorizationEndpoint, | |
| 173 | + tokenUrl: oauth.tokenEndpoint, | |
| 174 | + clientId: "test-client", | |
| 175 | + clientSecret: "test-secret", | |
| 176 | + originIntegration: IntegrationSlug.make(slug), | |
| 177 | + }, | |
| 178 | + }); | |
| 179 | + | |
| 180 | + const started = yield* client.oauth.start({ | |
| 181 | + payload: { | |
| 182 | + client: clientSlug, | |
| 183 | + clientOwner: "org", | |
| 184 | + owner: "org", | |
| 185 | + name: ConnectionName.make("main"), | |
| 186 | + integration: IntegrationSlug.make(slug), | |
| 187 | + template: AuthTemplateSlug.make("oauth"), | |
| 188 | + }, | |
| 189 | + }); | |
| 190 | + expect(started.status, "oauth.start redirects to the authorization server").toBe( | |
| 191 | + "redirect", | |
| 192 | + ); | |
| 193 | + if (started.status !== "redirect") return yield* Effect.die("no redirect"); | |
| 194 | + | |
| 195 | + // Drive the test IdP's consent by hand (authorize → login → code). | |
| 196 | + const code = yield* Effect.promise(async () => { | |
| 197 | + const authorize = await fetch(started.authorizationUrl, { redirect: "manual" }); | |
| 198 | + const loginUrl = authorize.headers.get("location"); | |
| 199 | + if (!loginUrl) throw new Error(`authorize did not redirect: ${authorize.status}`); | |
| 200 | + const login = await fetch(loginUrl, { | |
| 201 | + method: "POST", | |
| 202 | + headers: { | |
| 203 | + authorization: `Basic ${Buffer.from("alice:password").toString("base64")}`, | |
| 204 | + }, | |
| 205 | + redirect: "manual", | |
| 206 | + }); | |
| 207 | + const callbackUrl = login.headers.get("location"); | |
| 208 | + if (!callbackUrl) throw new Error(`login did not redirect: ${login.status}`); | |
| 209 | + const minted = new URL(callbackUrl).searchParams.get("code"); | |
| 210 | + if (!minted) throw new Error("callback carried no authorization code"); | |
| 211 | + return minted; | |
| 212 | + }); | |
| 213 | + yield* client.oauth.complete({ payload: { state: started.state, code } }); | |
| 214 | + | |
| 215 | + const tools = yield* client.tools.list({ query: {} }); | |
| 216 | + const address = tools | |
| 217 | + .filter((tool) => String(tool.integration) === slug) | |
| 218 | + .map((tool) => String(tool.address)) | |
| 219 | + .find((addr) => addr.endsWith("listIssues")); | |
| 220 | + expect(address, "the listIssues tool is in the catalog").toBeDefined(); | |
| 221 | + | |
| 222 | + // Call through the real MCP surface, the channel the reported | |
| 223 | + // failure was seen on. | |
| 224 | + const session = mcp.session(identity); | |
| 225 | + let called = yield* session.call("execute", { | |
| 226 | + code: invokeByAddressCode(address!, {}), | |
| 227 | + }); | |
| 228 | + // Approval-gated tools pause the execution once per gated call. | |
| 229 | + let guard = 0; | |
| 230 | + while (called.text.includes("executionId:") && guard < 10) { | |
| 231 | + called = yield* session.approvePaused(called.text); | |
| 232 | + guard += 1; | |
| 233 | + } | |
| 234 | + expect( | |
| 235 | + called.ok, | |
| 236 | + `the MCP execute call itself completed (got: ${called.text.slice(0, 400)})`, | |
| 237 | + ).toBe(true); | |
| 238 | + const envelope = JSON.parse(called.text) as ToolEnvelope; | |
| 239 | + | |
| 240 | + // THE guarantee: the caller gets data, not `oauth_refresh_failed`. | |
| 241 | + // The access token had already expired, so this call could only | |
| 242 | + // succeed by refreshing. | |
| 243 | + expect( | |
| 244 | + envelope.ok, | |
| 245 | + `the tool call succeeded (got: ${JSON.stringify(envelope.error ?? {}).slice(0, 400)})`, | |
| 246 | + ).toBe(true); | |
| 247 | + expect( | |
| 248 | + JSON.stringify(envelope.data ?? {}), | |
| 249 | + "the upstream's payload came back, so the retried token really worked upstream", | |
| 250 | + ).toContain("issue-1"); | |
| 251 | + | |
| 252 | + // Proven from the AS's own ledger: the scope-bearing grant was | |
| 253 | + // refused, and the scope-less retry is what succeeded. | |
| 254 | + const refreshGrants = (yield* oauth.requests).filter( | |
| 255 | + (request) => | |
| 256 | + request.path === "/token" && | |
| 257 | + request.method === "POST" && | |
| 258 | + request.body.includes("grant_type=refresh_token"), | |
| 259 | + ); | |
| 260 | + expect( | |
| 261 | + refreshGrants.length, | |
| 262 | + "the refused refresh and the retry are both on the ledger (2 requests)", | |
| 263 | + ).toBe(2); | |
| 264 | + const refused = refreshGrants[0]!; | |
| 265 | + const retried = refreshGrants[1]!; | |
| 266 | + expect( | |
| 267 | + new URLSearchParams(refused.body).get("scope"), | |
| 268 | + "the first refresh echoed the grant the connection recorded", | |
| 269 | + ).toBe("issues.read issues.write"); | |
| 270 | + expect( | |
| 271 | + new URLSearchParams(retried.body).has("scope"), | |
| 272 | + "the retry omitted scope, the form RFC 6749 §6 defines as 'the grant originally issued'", | |
| 273 | + ).toBe(false); | |
| 274 | + expect( | |
| 275 | + new URLSearchParams(retried.body).get("refresh_token"), | |
| 276 | + "the retry replayed the same still-live refresh token", | |
| 277 | + ).toBe(new URLSearchParams(refused.body).get("refresh_token")); | |
| 278 | + }), | |
| 279 | + Effect.gen(function* () { | |
| 280 | + yield* client.connections | |
| 281 | + .remove({ | |
| 282 | + params: { | |
| 283 | + owner: "org", | |
| 284 | + integration: IntegrationSlug.make(slug), | |
| 285 | + name: ConnectionName.make("main"), | |
| 286 | + }, | |
| 287 | + }) | |
| 288 | + .pipe(Effect.ignore); | |
| 289 | + yield* client.oauth | |
| 290 | + .removeClient({ params: { slug: clientSlug }, payload: { owner: "org" } }) | |
| 291 | + .pipe(Effect.ignore); | |
| 292 | + yield* client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore); | |
| 293 | + }), | |
| 294 | + ); | |
| 295 | + }), | |
| 296 | + ), | |
| 297 | +); | |