| @@ -0,0 +1,531 @@ |
| 1 | +// Cross-target: a connection whose credential cannot be resolved is a health |
| 2 | +// VERDICT, not a failed request — and the verdict is persisted, so the next |
| 3 | +// surface to ask reads it instead of hammering the authorization server. |
| 4 | +// |
| 5 | +// Production symptom: a handful of connections whose authorization server had |
| 6 | +// stopped re-minting credentials produced hundreds of server errors, every one |
| 7 | +// of them on `/api/connections/.../health`, plus unbounded refresh traffic to |
| 8 | +// the third party. A probe whose credential resolution failed escaped before |
| 9 | +// the verdict was written, so nothing was ever persisted, the freshness gate |
| 10 | +// had nothing to serve, and every mount of every surface sent another refresh |
| 11 | +// grant to a server that was already refusing. |
| 12 | +// |
| 13 | +// The journey: an OpenAPI integration completes a real authorization-code flow |
| 14 | +// against a live test AS that mints instantly-expiring access tokens and |
| 15 | +// refuses every refresh grant. A health check is configured on the |
| 16 | +// integration, so the connection takes the probing path. The probe cannot get |
| 17 | +// a credential — and must answer with a verdict, persist it, and let a |
| 18 | +// freshness window keep the next check off the wire. Both refusals are |
| 19 | +// covered: a retryable one (`degraded`) and a dead grant (`expired`). |
| 20 | +// |
| 21 | +// A second scenario in this file walks the same broken connection through the |
| 22 | +// real UI, because the other half of the symptom was the number of SURFACES: |
| 23 | +// see its header. |
| 24 | +import { randomBytes } from "node:crypto"; |
| 25 | +import { createServer } from "node:http"; |
| 26 | + |
| 27 | +import { expect } from "@effect/vitest"; |
| 28 | +import { Effect } from "effect"; |
| 29 | +import type { HttpApiClient } from "effect/unstable/httpapi"; |
| 30 | +import type { Response } from "playwright"; |
| 31 | +import { composePluginApi } from "@executor-js/api/server"; |
| 32 | +import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; |
| 33 | +import { |
| 34 | + AuthTemplateSlug, |
| 35 | + ConnectionName, |
| 36 | + IntegrationSlug, |
| 37 | + OAuthClientSlug, |
| 38 | +} from "@executor-js/sdk/shared"; |
| 39 | +import { serveOAuthTestServer } from "@executor-js/sdk/testing"; |
| 40 | + |
| 41 | +import { scenario } from "../src/scenario"; |
| 42 | +import { Api, Browser, Target } from "../src/services"; |
| 43 | +import { visit } from "../src/surfaces/browser"; |
| 44 | + |
| 45 | +const api = composePluginApi([openApiHttpPlugin()] as const); |
| 46 | +type Client = HttpApiClient.ForApi<typeof api>; |
| 47 | + |
| 48 | +const unique = (prefix: string) => `${prefix}${randomBytes(4).toString("hex")}`; |
| 49 | + |
| 50 | +/** Upstream on 127.0.0.1 whose `GET /me` is the obvious health probe. The |
| 51 | + * credential never resolves in this scenario, so a request reaching here at |
| 52 | + * all would mean the probe ran with no token — the 401 keeps that honest. */ |
| 53 | +const serveUpstream = () => |
| 54 | + Effect.acquireRelease( |
| 55 | + Effect.callback<{ readonly url: string; readonly close: () => void }>((resume) => { |
| 56 | + const server = createServer((request, response) => { |
| 57 | + if (request.method === "GET" && (request.url ?? "").startsWith("/me")) { |
| 58 | + const authorized = (request.headers["authorization"] ?? "").startsWith("Bearer at_"); |
| 59 | + response.writeHead(authorized ? 200 : 401, { |
| 60 | + "content-type": "application/json", |
| 61 | + }); |
| 62 | + response.end(JSON.stringify(authorized ? { email: "probe@example.test" } : {})); |
| 63 | + return; |
| 64 | + } |
| 65 | + response.writeHead(404, { "content-type": "application/json" }); |
| 66 | + response.end(JSON.stringify({ error: "not_found" })); |
| 67 | + }); |
| 68 | + server.listen(0, "127.0.0.1", () => { |
| 69 | + const address = server.address(); |
| 70 | + const port = typeof address === "object" && address ? address.port : 0; |
| 71 | + resume( |
| 72 | + Effect.succeed({ |
| 73 | + url: `http://127.0.0.1:${port}`, |
| 74 | + close: () => { |
| 75 | + server.close(); |
| 76 | + server.closeAllConnections(); |
| 77 | + }, |
| 78 | + }), |
| 79 | + ); |
| 80 | + }); |
| 81 | + }), |
| 82 | + (server) => Effect.sync(server.close), |
| 83 | + ); |
| 84 | + |
| 85 | +const spec = ( |
| 86 | + baseUrl: string, |
| 87 | + oauth: { |
| 88 | + readonly authorizationEndpoint: string; |
| 89 | + readonly tokenEndpoint: string; |
| 90 | + }, |
| 91 | +): string => |
| 92 | + JSON.stringify({ |
| 93 | + openapi: "3.0.3", |
| 94 | + info: { title: "Identity API", version: "1.0.0" }, |
| 95 | + servers: [{ url: baseUrl }], |
| 96 | + paths: { |
| 97 | + "/me": { |
| 98 | + get: { |
| 99 | + operationId: "getMe", |
| 100 | + summary: "The current account", |
| 101 | + security: [{ oauth: ["identity.read"] }], |
| 102 | + responses: { "200": { description: "account" } }, |
| 103 | + }, |
| 104 | + }, |
| 105 | + }, |
| 106 | + components: { |
| 107 | + securitySchemes: { |
| 108 | + oauth: { |
| 109 | + type: "oauth2", |
| 110 | + flows: { |
| 111 | + authorizationCode: { |
| 112 | + authorizationUrl: oauth.authorizationEndpoint, |
| 113 | + tokenUrl: oauth.tokenEndpoint, |
| 114 | + scopes: { "identity.read": "Read the account" }, |
| 115 | + }, |
| 116 | + }, |
| 117 | + }, |
| 118 | + }, |
| 119 | + }, |
| 120 | + }); |
| 121 | + |
| 122 | +/** One integration with a declared health check, one OAuth client, one |
| 123 | + * connection completed through a real authorization-code flow — against an |
| 124 | + * authorization server that refuses every refresh grant in the given way. |
| 125 | + * Instantly-expiring access tokens mean every credential resolution must |
| 126 | + * refresh, so the refusal is what the probe meets. */ |
| 127 | +const connectRefusing = ( |
| 128 | + client: Client, |
| 129 | + upstream: { readonly url: string }, |
| 130 | + options: { |
| 131 | + readonly name: ConnectionName; |
| 132 | + readonly errorCode: string; |
| 133 | + readonly description: string; |
| 134 | + }, |
| 135 | +) => |
| 136 | + Effect.gen(function* () { |
| 137 | + const oauth = yield* serveOAuthTestServer({ |
| 138 | + scopes: ["identity.read"], |
| 139 | + tokenExpiresInSeconds: 0, |
| 140 | + supportRefresh: false, |
| 141 | + invalidRefreshTokenErrorCode: options.errorCode, |
| 142 | + invalidRefreshTokenDescription: options.description, |
| 143 | + }); |
| 144 | + const slug = IntegrationSlug.make(unique("healthverdict")); |
| 145 | + const clientSlug = OAuthClientSlug.make(unique("healthverdictc")); |
| 146 | + |
| 147 | + yield* Effect.addFinalizer(() => |
| 148 | + Effect.all( |
| 149 | + [ |
| 150 | + client.connections |
| 151 | + .remove({ |
| 152 | + params: { owner: "org", integration: slug, name: options.name }, |
| 153 | + }) |
| 154 | + .pipe(Effect.ignore), |
| 155 | + client.oauth |
| 156 | + .removeClient({ |
| 157 | + params: { slug: clientSlug }, |
| 158 | + payload: { owner: "org" }, |
| 159 | + }) |
| 160 | + .pipe(Effect.ignore), |
| 161 | + client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore), |
| 162 | + ], |
| 163 | + { discard: true }, |
| 164 | + ), |
| 165 | + ); |
| 166 | + |
| 167 | + yield* client.openapi.addSpec({ |
| 168 | + payload: { |
| 169 | + spec: { kind: "blob", value: spec(upstream.url, oauth) }, |
| 170 | + slug, |
| 171 | + baseUrl: upstream.url, |
| 172 | + authenticationTemplate: [ |
| 173 | + { |
| 174 | + slug: "oauth", |
| 175 | + kind: "oauth2", |
| 176 | + authorizationUrl: oauth.authorizationEndpoint, |
| 177 | + tokenUrl: oauth.tokenEndpoint, |
| 178 | + scopes: ["identity.read"], |
| 179 | + }, |
| 180 | + ], |
| 181 | + }, |
| 182 | + }); |
| 183 | + |
| 184 | + // Configure the probe, the way the user does in the editor: the |
| 185 | + // ranked candidates offer the identity GET, and picking it is what |
| 186 | + // sends this connection down the probing path. |
| 187 | + const candidates = yield* client.integrations.healthCheckCandidates({ |
| 188 | + params: { slug }, |
| 189 | + }); |
| 190 | + const getMe = candidates.find((candidate) => candidate.method === "get"); |
| 191 | + if (!getMe) return yield* Effect.die("the identity spec exposed no GET candidate"); |
| 192 | + yield* client.integrations.healthCheckSet({ |
| 193 | + params: { slug }, |
| 194 | + payload: { spec: { operation: getMe.operation, identityField: "email" } }, |
| 195 | + }); |
| 196 | + |
| 197 | + yield* client.oauth.createClient({ |
| 198 | + payload: { |
| 199 | + owner: "org", |
| 200 | + slug: clientSlug, |
| 201 | + grant: "authorization_code", |
| 202 | + authorizationUrl: oauth.authorizationEndpoint, |
| 203 | + tokenUrl: oauth.tokenEndpoint, |
| 204 | + clientId: "test-client", |
| 205 | + clientSecret: "test-secret", |
| 206 | + originIntegration: slug, |
| 207 | + }, |
| 208 | + }); |
| 209 | + |
| 210 | + const started = yield* client.oauth.start({ |
| 211 | + payload: { |
| 212 | + client: clientSlug, |
| 213 | + clientOwner: "org", |
| 214 | + owner: "org", |
| 215 | + name: options.name, |
| 216 | + integration: slug, |
| 217 | + template: AuthTemplateSlug.make("oauth"), |
| 218 | + }, |
| 219 | + }); |
| 220 | + expect(started.status, "oauth.start redirects to the authorization server").toBe("redirect"); |
| 221 | + if (started.status !== "redirect") return yield* Effect.die("no redirect"); |
| 222 | + |
| 223 | + // Drive the test IdP's consent by hand (authorize → login → code). |
| 224 | + const code = yield* Effect.promise(async () => { |
| 225 | + const authorize = await fetch(started.authorizationUrl, { |
| 226 | + redirect: "manual", |
| 227 | + }); |
| 228 | + const loginUrl = authorize.headers.get("location"); |
| 229 | + if (!loginUrl) throw new Error(`authorize did not redirect: ${authorize.status}`); |
| 230 | + const login = await fetch(loginUrl, { |
| 231 | + method: "POST", |
| 232 | + headers: { |
| 233 | + authorization: `Basic ${Buffer.from("alice:password").toString("base64")}`, |
| 234 | + }, |
| 235 | + redirect: "manual", |
| 236 | + }); |
| 237 | + const callbackUrl = login.headers.get("location"); |
| 238 | + if (!callbackUrl) throw new Error(`login did not redirect: ${login.status}`); |
| 239 | + const minted = new URL(callbackUrl).searchParams.get("code"); |
| 240 | + if (!minted) throw new Error("callback carried no authorization code"); |
| 241 | + return minted; |
| 242 | + }); |
| 243 | + yield* client.oauth.complete({ payload: { state: started.state, code } }); |
| 244 | + yield* oauth.clearRequests; |
| 245 | + |
| 246 | + return { |
| 247 | + slug, |
| 248 | + /** Refresh grants the authorization server has actually received. |
| 249 | + * This is the number the whole fix is about: probing must not |
| 250 | + * scale with the number of surfaces asking. */ |
| 251 | + refreshGrants: oauth.requests.pipe( |
| 252 | + Effect.map( |
| 253 | + (all) => |
| 254 | + all.filter( |
| 255 | + (request) => |
| 256 | + request.path === "/token" && |
| 257 | + request.method === "POST" && |
| 258 | + request.body.includes("grant_type=refresh_token"), |
| 259 | + ).length, |
| 260 | + ), |
| 261 | + ), |
| 262 | + }; |
| 263 | + }); |
| 264 | + |
| 265 | +scenario( |
| 266 | + "Health checks · a connection whose refresh is refused reports a persisted verdict instead of failing the request", |
| 267 | + {}, |
| 268 | + Effect.scoped( |
| 269 | + Effect.gen(function* () { |
| 270 | + const target = yield* Target; |
| 271 | + const { client: makeClient } = yield* Api; |
| 272 | + const identity = yield* target.newIdentity(); |
| 273 | + const client = yield* makeClient(api, identity); |
| 274 | + const upstream = yield* serveUpstream(); |
| 275 | + |
| 276 | + // ── A refusal that is not "re-auth required" ──────────────────────── |
| 277 | + // The AS answers the refresh with a code OTHER than invalid_grant, so |
| 278 | + // retrying could in principle work. This is the shape that used to |
| 279 | + // escape the probe as a server error. |
| 280 | + const degradedName = ConnectionName.make("healthverdictrefused"); |
| 281 | + const refused = yield* connectRefusing(client, upstream, { |
| 282 | + name: degradedName, |
| 283 | + errorCode: "invalid_request", |
| 284 | + description: "Refresh temporarily unavailable", |
| 285 | + }); |
| 286 | + |
| 287 | + // THE guarantee: a probe that cannot resolve its credential answers with |
| 288 | + // a verdict. The request itself succeeds — a third party refusing a |
| 289 | + // refresh is not a defect in this product, and reporting it as one is |
| 290 | + // what buried the real signal. |
| 291 | + const probed = yield* client.connections.checkHealth({ |
| 292 | + params: { owner: "org", integration: refused.slug, name: degradedName }, |
| 293 | + query: {}, |
| 294 | + }); |
| 295 | + expect( |
| 296 | + probed.status, |
| 297 | + "a connection whose refresh is refused reads degraded, not a failed request", |
| 298 | + ).toBe("degraded"); |
| 299 | + expect( |
| 300 | + probed.detail ?? "", |
| 301 | + "the verdict carries the authorization server's reason", |
| 302 | + ).toContain("Refresh temporarily unavailable"); |
| 303 | + expect(yield* refused.refreshGrants, "the probe did try the refresh exactly once").toBe(1); |
| 304 | + |
| 305 | + // The verdict PERSISTS, so the accounts list shows the state at a glance |
| 306 | + // and the freshness gate has something to serve. |
| 307 | + const stored = yield* client.connections.get({ |
| 308 | + params: { owner: "org", integration: refused.slug, name: degradedName }, |
| 309 | + }); |
| 310 | + expect(stored?.lastHealth?.status, "the verdict is persisted on the connection").toBe( |
| 311 | + "degraded", |
| 312 | + ); |
| 313 | + expect(stored?.lastHealth?.checkedAt, "and it is the verdict this probe produced").toBe( |
| 314 | + probed.checkedAt, |
| 315 | + ); |
| 316 | + |
| 317 | + // And because it persisted, a caller that DOES pass a freshness window is |
| 318 | + // served from it: the server's gate has something to answer with, so the |
| 319 | + // authorization server sees nothing more. (The automatic client path |
| 320 | + // deliberately passes no window for a non-healthy verdict — recovery has |
| 321 | + // to be able to show — but the gate itself must work for the callers that |
| 322 | + // opt into it.) |
| 323 | + for (let mount = 0; mount < 3; mount++) { |
| 324 | + const remount = yield* client.connections.checkHealth({ |
| 325 | + params: { |
| 326 | + owner: "org", |
| 327 | + integration: refused.slug, |
| 328 | + name: degradedName, |
| 329 | + }, |
| 330 | + query: { ifStaleMs: 30_000 }, |
| 331 | + }); |
| 332 | + expect(remount.status, "a repeat check inside the window keeps the verdict").toBe( |
| 333 | + "degraded", |
| 334 | + ); |
| 335 | + expect(remount.checkedAt, "and it IS the persisted verdict, not a new probe").toBe( |
| 336 | + probed.checkedAt, |
| 337 | + ); |
| 338 | + } |
| 339 | + expect( |
| 340 | + yield* refused.refreshGrants, |
| 341 | + "repeated checks inside the window never reach the authorization server again", |
| 342 | + ).toBe(1); |
| 343 | + |
| 344 | + // ── A dead grant ─────────────────────────────────────────────────── |
| 345 | + // invalid_grant means the user must re-authenticate: a different verdict |
| 346 | + // through the same probing path, and it persists the same way. |
| 347 | + const expiredName = ConnectionName.make("healthverdictdead"); |
| 348 | + const revoked = yield* connectRefusing(client, upstream, { |
| 349 | + name: expiredName, |
| 350 | + errorCode: "invalid_grant", |
| 351 | + description: "Grant revoked", |
| 352 | + }); |
| 353 | + |
| 354 | + const dead = yield* client.connections.checkHealth({ |
| 355 | + params: { owner: "org", integration: revoked.slug, name: expiredName }, |
| 356 | + query: {}, |
| 357 | + }); |
| 358 | + expect(dead.status, "a revoked grant reads expired, so the UI can offer reconnect").toBe( |
| 359 | + "expired", |
| 360 | + ); |
| 361 | + expect(dead.detail ?? "", "the verdict carries the authorization server's reason").toContain( |
| 362 | + "Grant revoked", |
| 363 | + ); |
| 364 | + const storedDead = yield* client.connections.get({ |
| 365 | + params: { owner: "org", integration: revoked.slug, name: expiredName }, |
| 366 | + }); |
| 367 | + expect(storedDead?.lastHealth?.status, "the expired verdict is persisted too").toBe( |
| 368 | + "expired", |
| 369 | + ); |
| 370 | + }), |
| 371 | + ), |
| 372 | +); |
| 373 | + |
| 374 | +// =========================================================================== |
| 375 | +// The other half of the production symptom, through the real UI. A connection |
| 376 | +// whose refresh is refused is rendered by several surfaces (the integration |
| 377 | +// page's Connections list and the integrations list summary), and each one |
| 378 | +// revalidates on mount. Before the fix every one of those probes escaped as a |
| 379 | +// SERVER ERROR instead of a verdict, so one broken connection produced a |
| 380 | +// stream of captured errors and no surface could show the user what was wrong. |
| 381 | +// |
| 382 | +// What this pins down is that shape, not the volume: every automatic health |
| 383 | +// request succeeds and every surface paints the same Degraded verdict, while |
| 384 | +// the per-connection guard keeps each surface to exactly ONE probe — the |
| 385 | +// no-probe-storm invariant. Re-probing a broken connection once per surface is |
| 386 | +// the deliberate price of letting recovery show on the next load; an |
| 387 | +// unbounded loop is the regression worth catching. |
| 388 | +// |
| 389 | +// Nothing here touches the hook: the browser navigates between the two |
| 390 | +// surfaces a user actually visits, and the authorization server's own request |
| 391 | +// ledger is the judge. Skips on targets with no browser surface. |
| 392 | +// =========================================================================== |
| 393 | + |
| 394 | +scenario( |
| 395 | + "Health checks (UI) · every surface showing a broken connection paints a verdict, and probes exactly once", |
| 396 | + {}, |
| 397 | + Effect.scoped( |
| 398 | + Effect.gen(function* () { |
| 399 | + const target = yield* Target; |
| 400 | + const browser = yield* Browser; |
| 401 | + const { client: makeClient } = yield* Api; |
| 402 | + const identity = yield* target.newIdentity(); |
| 403 | + const client = yield* makeClient(api, identity); |
| 404 | + const upstream = yield* serveUpstream(); |
| 405 | + |
| 406 | + const name = ConnectionName.make("healthverdictui"); |
| 407 | + const refused = yield* connectRefusing(client, upstream, { |
| 408 | + name, |
| 409 | + errorCode: "invalid_request", |
| 410 | + description: "Refresh temporarily unavailable", |
| 411 | + }); |
| 412 | + |
| 413 | + yield* browser.session(identity, async ({ page, step }) => { |
| 414 | + const connections = page.locator("section").filter({ |
| 415 | + has: page.getByRole("heading", { level: 3, name: "Connections" }), |
| 416 | + }); |
| 417 | + // Every health request the app sends, with the status it came back |
| 418 | + // with: the client's own wire contract, observed from outside. A |
| 419 | + // credential the third party refuses must still produce a SUCCESSFUL |
| 420 | + // health response carrying a verdict — that is the whole fix. |
| 421 | + const healthRequests: string[] = []; |
| 422 | + const healthFailures: string[] = []; |
| 423 | + page.on("request", (request) => { |
| 424 | + if (request.method() === "POST" && request.url().includes("/health")) { |
| 425 | + healthRequests.push(request.url()); |
| 426 | + } |
| 427 | + }); |
| 428 | + page.on("response", (response) => { |
| 429 | + if ( |
| 430 | + response.request().method() === "POST" && |
| 431 | + response.url().includes("/health") && |
| 432 | + response.status() >= 400 |
| 433 | + ) { |
| 434 | + healthFailures.push(`${String(response.status())} ${response.url()}`); |
| 435 | + } |
| 436 | + }); |
| 437 | + const isHealthResponse = (response: Response) => |
| 438 | + response.request().method() === "POST" && response.url().includes("/health"); |
| 439 | + const refreshGrants = () => Effect.runPromise(refused.refreshGrants); |
| 440 | + |
| 441 | + // Surface 1. No verdict is persisted yet, so this load's automatic |
| 442 | + // check is the probe that discovers the broken credential — the badge |
| 443 | + // appearing is proof it completed and was written down. |
| 444 | + await step("Open the integration: the broken connection reads Degraded", async () => { |
| 445 | + const settled = page.waitForResponse(isHealthResponse, { |
| 446 | + timeout: 30_000, |
| 447 | + }); |
| 448 | + await visit(page, `/integrations/${refused.slug}`); |
| 449 | + await settled; |
| 450 | + await connections.getByText("Degraded", { exact: true }).waitFor({ timeout: 30_000 }); |
| 451 | + }); |
| 452 | + |
| 453 | + // The baseline is taken AFTER that probe, so the freshness window is |
| 454 | + // running from a verdict written moments ago and the assertion below |
| 455 | + // does not depend on how long the browser took to start. |
| 456 | + const baseline = await refreshGrants(); |
| 457 | + expect(baseline, "the first surface did probe the authorization server").toBeGreaterThan(0); |
| 458 | + const afterFirstSurface = healthRequests.length; |
| 459 | + |
| 460 | + // Surface 2: the integrations list summarises the same connection's |
| 461 | + // health, and mounts its own revalidation. |
| 462 | + await step("Open the integrations list, which shows the same connection", async () => { |
| 463 | + const settled = page.waitForResponse(isHealthResponse, { |
| 464 | + timeout: 30_000, |
| 465 | + }); |
| 466 | + await visit(page, "/"); |
| 467 | + await settled; |
| 468 | + }); |
| 469 | + |
| 470 | + // What the LIST surface cost at the authorization server. |
| 471 | + const afterList = await refreshGrants(); |
| 472 | + |
| 473 | + // Surface 3: back to the integration page — a fresh mount again. |
| 474 | + await step("Return to the integration page: another mount, another ask", async () => { |
| 475 | + const settled = page.waitForResponse(isHealthResponse, { |
| 476 | + timeout: 30_000, |
| 477 | + }); |
| 478 | + await visit(page, `/integrations/${refused.slug}`); |
| 479 | + await settled; |
| 480 | + await connections.getByText("Degraded", { exact: true }).waitFor({ timeout: 30_000 }); |
| 481 | + }); |
| 482 | + |
| 483 | + const afterReturn = await refreshGrants(); |
| 484 | + |
| 485 | + const laterRequests = healthRequests.slice(afterFirstSurface); |
| 486 | + expect( |
| 487 | + laterRequests.length, |
| 488 | + "the later surfaces did ask about the connection's health — silence would mean a broken connection could never be seen to recover", |
| 489 | + ).toBeGreaterThan(0); |
| 490 | + |
| 491 | + expect( |
| 492 | + afterList - baseline, |
| 493 | + "the list surface revalidated the broken connection", |
| 494 | + ).toBeGreaterThan(0); |
| 495 | + |
| 496 | + // THE production symptom, stated as the third party experiences it. |
| 497 | + // Not "the later surfaces went silent" — they must not, or recovery |
| 498 | + // could never show — but "the traffic is attributable and it stops". |
| 499 | + // |
| 500 | + // Attributable: a refresh grant only ever happens inside a health |
| 501 | + // request, so grants can never outnumber the requests that caused |
| 502 | + // them. More grants than requests is the server retrying in a loop. |
| 503 | + // Counted this way the assertion cannot race a probe that is still in |
| 504 | + // flight: the request is recorded when it is SENT, before its grant. |
| 505 | + expect( |
| 506 | + afterReturn, |
| 507 | + "every refresh grant is attributable to a health request the client sent", |
| 508 | + ).toBeLessThanOrEqual(healthRequests.length); |
| 509 | + |
| 510 | + // And it stops: once the page has settled, the per-connection guard |
| 511 | + // means no further probes are issued. A quiet window that stays quiet |
| 512 | + // is the storm's absence stated directly, rather than a probe count |
| 513 | + // sampled at one arbitrary instant. |
| 514 | + const settled = healthRequests.length; |
| 515 | + await page.waitForTimeout(3_000); |
| 516 | + expect( |
| 517 | + healthRequests.length, |
| 518 | + "a settled page stops probing; a still-climbing count is the guard failing and the effect re-probing in a loop", |
| 519 | + ).toBe(settled); |
| 520 | + |
| 521 | + // And the shape that actually reached production: a refused refresh is |
| 522 | + // answered, not raised. Every one of these requests used to come back |
| 523 | + // as a server error, which is what buried the signal. |
| 524 | + expect( |
| 525 | + healthFailures, |
| 526 | + "no automatic health request failed; a refused refresh is a verdict, not an error", |
| 527 | + ).toEqual([]); |
| 528 | + }); |
| 529 | + }), |
| 530 | + ), |
| 531 | +); |