| @@ -0,0 +1,297 @@ |
| | 1 | // Cross-target (browser): the health-probe churn loop seen in production. |
| | 2 | // |
| | 3 | // Production symptom (2026-09-18): one signed-in browser with many |
| | 4 | // non-healthy or never-checked connections spread across many integrations |
| | 5 | // sent hundreds of `/api/connections/.../health` POSTs and `/api/connections` |
| | 6 | // GETs per minute for as long as the dashboard stayed open. Most were |
| | 7 | // interrupted client-side before completing (the shared health mutation atom |
| | 8 | // cancels the previous in-flight call), but hundreds per minute still reached |
| | 9 | // the server and the upstreams. The server annotated every verdict as |
| | 10 | // unchanged, so the loop lives in the client. |
| | 11 | // |
| | 12 | // The existing verdict scenario pins "one broken connection probes exactly |
| | 13 | // once per surface". This one reproduces the production SHAPE: many |
| | 14 | // connections, several integrations, a mix of verdicts (`unknown` from a |
| | 15 | // connection with no probe configured, `degraded` from an upstream that |
| | 16 | // answers 401), and the page journey a user actually makes (integrations |
| | 17 | // list → integration page → integrations list). It then watches a quiet |
| | 18 | // window. A settled page must stop asking. A count that keeps climbing is the |
| | 19 | // loop. |
| | 20 | // |
| | 21 | // Skips on targets with no browser surface. |
| | 22 | import { randomBytes } from "node:crypto"; |
| | 23 | import { createServer } from "node:http"; |
| | 24 | |
| | 25 | import { expect } from "@effect/vitest"; |
| | 26 | import { Effect } from "effect"; |
| | 27 | import type { HttpApiClient } from "effect/unstable/httpapi"; |
| | 28 | import { composePluginApi } from "@executor-js/api/server"; |
| | 29 | import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; |
| | 30 | import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared"; |
| | 31 | |
| | 32 | import { scenario } from "../src/scenario"; |
| | 33 | import { Api, Browser, Target } from "../src/services"; |
| | 34 | import { visit } from "../src/surfaces/browser"; |
| | 35 | |
| | 36 | const api = composePluginApi([openApiHttpPlugin()] as const); |
| | 37 | type Client = HttpApiClient.ForApi<typeof api>; |
| | 38 | |
| | 39 | const TEMPLATE = AuthTemplateSlug.make("apiKey"); |
| | 40 | |
| | 41 | /** Integrations whose connection has a probe configured and an upstream that |
| | 42 | * rejects the key: the probe persists `degraded`. */ |
| | 43 | const DEGRADED_COUNT = 4; |
| | 44 | /** Integrations whose connection has NO probe configured: every automatic |
| | 45 | * check answers `unknown` with a fresh `checkedAt`. */ |
| | 46 | const UNKNOWN_COUNT = 4; |
| | 47 | |
| | 48 | /** How long a settled page is watched for further probes. Production cycled |
| | 49 | * every 0.5-1s, so a loop shows up well inside this. */ |
| | 50 | const QUIET_WINDOW_MS = 10_000; |
| | 51 | |
| | 52 | const unique = (prefix: string) => `${prefix}${randomBytes(3).toString("hex")}`; |
| | 53 | |
| | 54 | /** Upstream on 127.0.0.1 whose `GET /me` rejects every key. */ |
| | 55 | const serveRejectingUpstream = () => |
| | 56 | Effect.acquireRelease( |
| | 57 | Effect.callback<{ readonly url: string; readonly close: () => void }>((resume) => { |
| | 58 | const server = createServer((request, response) => { |
| | 59 | if (request.method === "GET" && (request.url ?? "").startsWith("/me")) { |
| | 60 | response.writeHead(401, { "content-type": "application/json" }); |
| | 61 | response.end(JSON.stringify({ error: "invalid_token" })); |
| | 62 | return; |
| | 63 | } |
| | 64 | response.writeHead(404, { "content-type": "application/json" }); |
| | 65 | response.end(JSON.stringify({ error: "not_found" })); |
| | 66 | }); |
| | 67 | server.listen(0, "127.0.0.1", () => { |
| | 68 | const address = server.address(); |
| | 69 | const port = typeof address === "object" && address ? address.port : 0; |
| | 70 | resume( |
| | 71 | Effect.succeed({ |
| | 72 | url: `http://127.0.0.1:${port}`, |
| | 73 | close: () => { |
| | 74 | server.close(); |
| | 75 | server.closeAllConnections(); |
| | 76 | }, |
| | 77 | }), |
| | 78 | ); |
| | 79 | }); |
| | 80 | }), |
| | 81 | (server) => Effect.sync(server.close), |
| | 82 | ); |
| | 83 | |
| | 84 | const identitySpec = (baseUrl: string, title: string): string => |
| | 85 | JSON.stringify({ |
| | 86 | openapi: "3.0.3", |
| | 87 | info: { title, version: "1.0.0" }, |
| | 88 | servers: [{ url: baseUrl }], |
| | 89 | paths: { |
| | 90 | "/me": { |
| | 91 | get: { |
| | 92 | operationId: "getMe", |
| | 93 | summary: "The current account", |
| | 94 | responses: { |
| | 95 | "200": { |
| | 96 | description: "The authenticated account", |
| | 97 | content: { |
| | 98 | "application/json": { |
| | 99 | schema: { type: "object", properties: { email: { type: "string" } } }, |
| | 100 | }, |
| | 101 | }, |
| | 102 | }, |
| | 103 | }, |
| | 104 | }, |
| | 105 | }, |
| | 106 | }, |
| | 107 | }); |
| | 108 | |
| | 109 | /** One OpenAPI integration with one saved org connection. With `probe`, the |
| | 110 | * identity GET is configured as the health check. */ |
| | 111 | const seedIntegration = ( |
| | 112 | client: Client, |
| | 113 | upstreamUrl: string, |
| | 114 | options: { readonly prefix: string; readonly probe: boolean }, |
| | 115 | ) => |
| | 116 | Effect.gen(function* () { |
| | 117 | const slug = IntegrationSlug.make(unique(options.prefix)); |
| | 118 | const name = ConnectionName.make(`${options.prefix}conn`); |
| | 119 | |
| | 120 | yield* Effect.addFinalizer(() => |
| | 121 | Effect.all( |
| | 122 | [ |
| | 123 | client.connections |
| | 124 | .remove({ params: { owner: "org", integration: slug, name } }) |
| | 125 | .pipe(Effect.ignore), |
| | 126 | client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore), |
| | 127 | ], |
| | 128 | { discard: true }, |
| | 129 | ), |
| | 130 | ); |
| | 131 | |
| | 132 | yield* client.openapi.addSpec({ |
| | 133 | payload: { |
| | 134 | spec: { kind: "blob", value: identitySpec(upstreamUrl, `Churn ${slug}`) }, |
| | 135 | slug, |
| | 136 | baseUrl: upstreamUrl, |
| | 137 | authenticationTemplate: [ |
| | 138 | { |
| | 139 | slug: "apiKey", |
| | 140 | type: "apiKey", |
| | 141 | headers: { authorization: ["Bearer ", { type: "variable", name: "token" }] }, |
| | 142 | }, |
| | 143 | ], |
| | 144 | }, |
| | 145 | }); |
| | 146 | |
| | 147 | if (options.probe) { |
| | 148 | const candidates = yield* client.integrations.healthCheckCandidates({ params: { slug } }); |
| | 149 | const getMe = candidates.find((candidate) => candidate.method === "get"); |
| | 150 | if (!getMe) return yield* Effect.die("identity spec exposed no GET candidate"); |
| | 151 | yield* client.integrations.healthCheckSet({ |
| | 152 | params: { slug }, |
| | 153 | payload: { spec: { operation: getMe.operation, identityField: "email" } }, |
| | 154 | }); |
| | 155 | } |
| | 156 | |
| | 157 | yield* client.connections.create({ |
| | 158 | payload: { owner: "org", name, integration: slug, template: TEMPLATE, value: "bad-key" }, |
| | 159 | }); |
| | 160 | |
| | 161 | return { slug, name }; |
| | 162 | }); |
| | 163 | |
| | 164 | scenario( |
| | 165 | "Health checks (UI) · many broken connections across many integrations settle instead of looping", |
| | 166 | {}, |
| | 167 | Effect.scoped( |
| | 168 | Effect.gen(function* () { |
| | 169 | const target = yield* Target; |
| | 170 | const browser = yield* Browser; |
| | 171 | const { client: makeClient } = yield* Api; |
| | 172 | const identity = yield* target.newIdentity(); |
| | 173 | const client = yield* makeClient(api, identity); |
| | 174 | const upstream = yield* serveRejectingUpstream(); |
| | 175 | |
| | 176 | const degraded = yield* Effect.all( |
| | 177 | Array.from({ length: DEGRADED_COUNT }, () => |
| | 178 | seedIntegration(client, upstream.url, { prefix: "churndeg", probe: true }), |
| | 179 | ), |
| | 180 | { concurrency: 2 }, |
| | 181 | ); |
| | 182 | const unknown = yield* Effect.all( |
| | 183 | Array.from({ length: UNKNOWN_COUNT }, () => |
| | 184 | seedIntegration(client, upstream.url, { prefix: "churnunk", probe: false }), |
| | 185 | ), |
| | 186 | { concurrency: 2 }, |
| | 187 | ); |
| | 188 | const seeded = [...degraded, ...unknown]; |
| | 189 | const connectionCount = seeded.length; |
| | 190 | |
| | 191 | // Persist a verdict on every connection BEFORE the browser opens, the |
| | 192 | // way a returning user finds them: `degraded` on the probed ones, |
| | 193 | // `unknown` on the rest. The list then renders each row with a |
| | 194 | // persisted verdict, and every automatic revalidation compares its |
| | 195 | // result against one. |
| | 196 | for (const { slug, name } of seeded) { |
| | 197 | yield* client.connections.checkHealth({ |
| | 198 | params: { owner: "org", integration: slug, name }, |
| | 199 | query: {}, |
| | 200 | }); |
| | 201 | } |
| | 202 | |
| | 203 | yield* browser.session(identity, async ({ page, step }) => { |
| | 204 | // The client's wire contract, observed from outside: every health POST |
| | 205 | // and every connections-list GET the app sends, plus how many the |
| | 206 | // browser reports as failed (an aborted request shows up here as a |
| | 207 | // failure, which is how the interrupted mutation atom looks on the wire). |
| | 208 | const health: string[] = []; |
| | 209 | const lists: string[] = []; |
| | 210 | let aborted = 0; |
| | 211 | // Only THIS scenario's connections count. Targets that share one org |
| | 212 | // across scenarios (selfhost) can carry rows another scenario left |
| | 213 | // behind, and those revalidate too; they are not ours to bound. |
| | 214 | const seededSlugs = new Set(seeded.map(({ slug }) => String(slug))); |
| | 215 | const isHealth = (method: string, url: string) => |
| | 216 | method === "POST" && |
| | 217 | url.includes("/health") && |
| | 218 | [...seededSlugs].some((slug) => url.includes(`/api/connections/org/${slug}/`)); |
| | 219 | const isList = (method: string, url: string) => |
| | 220 | method === "GET" && /\/api\/connections(\?|$)/.test(url); |
| | 221 | page.on("request", (request) => { |
| | 222 | const url = request.url(); |
| | 223 | if (isHealth(request.method(), url)) health.push(url); |
| | 224 | else if (isList(request.method(), url)) lists.push(url); |
| | 225 | }); |
| | 226 | page.on("requestfailed", (request) => { |
| | 227 | const url = request.url(); |
| | 228 | if (isHealth(request.method(), url) || isList(request.method(), url)) aborted += 1; |
| | 229 | }); |
| | 230 | |
| | 231 | const snapshot = (label: string) => { |
| | 232 | const line = `[churn] ${label}: health=${String(health.length)} lists=${String(lists.length)} aborted=${String(aborted)}`; |
| | 233 | console.log(line); |
| | 234 | return { health: health.length, lists: lists.length, aborted }; |
| | 235 | }; |
| | 236 | |
| | 237 | await step("Open the integrations list with every broken connection on it", async () => { |
| | 238 | await visit(page, "/"); |
| | 239 | for (const { slug } of seeded) { |
| | 240 | await page |
| | 241 | .getByRole("link", { name: new RegExp(slug, "i") }) |
| | 242 | .first() |
| | 243 | .waitFor({ |
| | 244 | timeout: 30_000, |
| | 245 | }); |
| | 246 | } |
| | 247 | // Let the automatic revalidation land for every row. |
| | 248 | await page.waitForTimeout(3_000); |
| | 249 | }); |
| | 250 | const afterList = snapshot("after integrations list"); |
| | 251 | |
| | 252 | await step("Open one integration page, then return to the list", async () => { |
| | 253 | await visit(page, `/integrations/${degraded[0]!.slug}`); |
| | 254 | await page.waitForTimeout(2_000); |
| | 255 | await visit(page, "/"); |
| | 256 | await page.waitForTimeout(3_000); |
| | 257 | }); |
| | 258 | const afterJourney = snapshot("after journey"); |
| | 259 | |
| | 260 | // The quiet window: nothing changed on the page, nothing changed on |
| | 261 | // the server, so nothing more should be asked. |
| | 262 | await step("Leave the settled list open and watch the wire", async () => { |
| | 263 | await page.waitForTimeout(QUIET_WINDOW_MS); |
| | 264 | }); |
| | 265 | const afterQuiet = snapshot("after quiet window"); |
| | 266 | |
| | 267 | const quietHealth = afterQuiet.health - afterJourney.health; |
| | 268 | const quietLists = afterQuiet.lists - afterJourney.lists; |
| | 269 | console.log( |
| | 270 | `[churn] quiet window (${String(QUIET_WINDOW_MS)}ms): +${String(quietHealth)} health, +${String(quietLists)} list fetches over ${String(connectionCount)} connections`, |
| | 271 | ); |
| | 272 | |
| | 273 | // One list mount revalidates each non-healthy connection once, and |
| | 274 | // reads the connections list once per owner (plus the unscoped read). |
| | 275 | // A probe count above the connection count means rows are being |
| | 276 | // re-probed; a list count above three means verdicts are being |
| | 277 | // treated as changes and refetching the list. |
| | 278 | expect( |
| | 279 | afterList.health, |
| | 280 | `the first list mount probes each broken connection at most once (sent ${String(afterList.health)} for ${String(connectionCount)} connections)`, |
| | 281 | ).toBeLessThanOrEqual(connectionCount); |
| | 282 | expect( |
| | 283 | afterList.lists, |
| | 284 | `the first list mount reads the connections list at most once per owner (sent ${String(afterList.lists)})`, |
| | 285 | ).toBeLessThanOrEqual(3); |
| | 286 | |
| | 287 | // THE production symptom: a settled page keeps asking. Zero is the |
| | 288 | // contract; anything else is the loop. |
| | 289 | expect(quietHealth, "a settled integrations list sends no further health probes").toBe(0); |
| | 290 | expect( |
| | 291 | quietLists, |
| | 292 | "a settled integrations list does not refetch the connections list", |
| | 293 | ).toBe(0); |
| | 294 | }); |
| | 295 | }), |
| | 296 | ), |
| | 297 | ); |