| @@ -0,0 +1,297 @@ |
| 1 | +// Cross-target (browser): the health-probe churn loop seen in production. |
| 2 | +// |
| 3 | +// Production symptom (2026-09-18): one signed-in browser with many |
| 4 | +// non-healthy or never-checked connections spread across many integrations |
| 5 | +// sent hundreds of `/api/connections/.../health` POSTs and `/api/connections` |
| 6 | +// GETs per minute for as long as the dashboard stayed open. Most were |
| 7 | +// interrupted client-side before completing (the shared health mutation atom |
| 8 | +// cancels the previous in-flight call), but hundreds per minute still reached |
| 9 | +// the server and the upstreams. The server annotated every verdict as |
| 10 | +// unchanged, so the loop lives in the client. |
| 11 | +// |
| 12 | +// The existing verdict scenario pins "one broken connection probes exactly |
| 13 | +// once per surface". This one reproduces the production SHAPE: many |
| 14 | +// connections, several integrations, a mix of verdicts (`unknown` from a |
| 15 | +// connection with no probe configured, `degraded` from an upstream that |
| 16 | +// answers 401), and the page journey a user actually makes (integrations |
| 17 | +// list → integration page → integrations list). It then watches a quiet |
| 18 | +// window. A settled page must stop asking. A count that keeps climbing is the |
| 19 | +// loop. |
| 20 | +// |
| 21 | +// Skips on targets with no browser surface. |
| 22 | +import { randomBytes } from "node:crypto"; |
| 23 | +import { createServer } from "node:http"; |
| 24 | + |
| 25 | +import { expect } from "@effect/vitest"; |
| 26 | +import { Effect } from "effect"; |
| 27 | +import type { HttpApiClient } from "effect/unstable/httpapi"; |
| 28 | +import { composePluginApi } from "@executor-js/api/server"; |
| 29 | +import { openApiHttpPlugin } from "@executor-js/plugin-openapi/api"; |
| 30 | +import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared"; |
| 31 | + |
| 32 | +import { scenario } from "../src/scenario"; |
| 33 | +import { Api, Browser, Target } from "../src/services"; |
| 34 | +import { visit } from "../src/surfaces/browser"; |
| 35 | + |
| 36 | +const api = composePluginApi([openApiHttpPlugin()] as const); |
| 37 | +type Client = HttpApiClient.ForApi<typeof api>; |
| 38 | + |
| 39 | +const TEMPLATE = AuthTemplateSlug.make("apiKey"); |
| 40 | + |
| 41 | +/** Integrations whose connection has a probe configured and an upstream that |
| 42 | + * rejects the key: the probe persists `degraded`. */ |
| 43 | +const DEGRADED_COUNT = 4; |
| 44 | +/** Integrations whose connection has NO probe configured: every automatic |
| 45 | + * check answers `unknown` with a fresh `checkedAt`. */ |
| 46 | +const UNKNOWN_COUNT = 4; |
| 47 | + |
| 48 | +/** How long a settled page is watched for further probes. Production cycled |
| 49 | + * every 0.5-1s, so a loop shows up well inside this. */ |
| 50 | +const QUIET_WINDOW_MS = 10_000; |
| 51 | + |
| 52 | +const unique = (prefix: string) => `${prefix}${randomBytes(3).toString("hex")}`; |
| 53 | + |
| 54 | +/** Upstream on 127.0.0.1 whose `GET /me` rejects every key. */ |
| 55 | +const serveRejectingUpstream = () => |
| 56 | + Effect.acquireRelease( |
| 57 | + Effect.callback<{ readonly url: string; readonly close: () => void }>((resume) => { |
| 58 | + const server = createServer((request, response) => { |
| 59 | + if (request.method === "GET" && (request.url ?? "").startsWith("/me")) { |
| 60 | + response.writeHead(401, { "content-type": "application/json" }); |
| 61 | + response.end(JSON.stringify({ error: "invalid_token" })); |
| 62 | + return; |
| 63 | + } |
| 64 | + response.writeHead(404, { "content-type": "application/json" }); |
| 65 | + response.end(JSON.stringify({ error: "not_found" })); |
| 66 | + }); |
| 67 | + server.listen(0, "127.0.0.1", () => { |
| 68 | + const address = server.address(); |
| 69 | + const port = typeof address === "object" && address ? address.port : 0; |
| 70 | + resume( |
| 71 | + Effect.succeed({ |
| 72 | + url: `http://127.0.0.1:${port}`, |
| 73 | + close: () => { |
| 74 | + server.close(); |
| 75 | + server.closeAllConnections(); |
| 76 | + }, |
| 77 | + }), |
| 78 | + ); |
| 79 | + }); |
| 80 | + }), |
| 81 | + (server) => Effect.sync(server.close), |
| 82 | + ); |
| 83 | + |
| 84 | +const identitySpec = (baseUrl: string, title: string): string => |
| 85 | + JSON.stringify({ |
| 86 | + openapi: "3.0.3", |
| 87 | + info: { title, version: "1.0.0" }, |
| 88 | + servers: [{ url: baseUrl }], |
| 89 | + paths: { |
| 90 | + "/me": { |
| 91 | + get: { |
| 92 | + operationId: "getMe", |
| 93 | + summary: "The current account", |
| 94 | + responses: { |
| 95 | + "200": { |
| 96 | + description: "The authenticated account", |
| 97 | + content: { |
| 98 | + "application/json": { |
| 99 | + schema: { type: "object", properties: { email: { type: "string" } } }, |
| 100 | + }, |
| 101 | + }, |
| 102 | + }, |
| 103 | + }, |
| 104 | + }, |
| 105 | + }, |
| 106 | + }, |
| 107 | + }); |
| 108 | + |
| 109 | +/** One OpenAPI integration with one saved org connection. With `probe`, the |
| 110 | + * identity GET is configured as the health check. */ |
| 111 | +const seedIntegration = ( |
| 112 | + client: Client, |
| 113 | + upstreamUrl: string, |
| 114 | + options: { readonly prefix: string; readonly probe: boolean }, |
| 115 | +) => |
| 116 | + Effect.gen(function* () { |
| 117 | + const slug = IntegrationSlug.make(unique(options.prefix)); |
| 118 | + const name = ConnectionName.make(`${options.prefix}conn`); |
| 119 | + |
| 120 | + yield* Effect.addFinalizer(() => |
| 121 | + Effect.all( |
| 122 | + [ |
| 123 | + client.connections |
| 124 | + .remove({ params: { owner: "org", integration: slug, name } }) |
| 125 | + .pipe(Effect.ignore), |
| 126 | + client.openapi.removeSpec({ params: { slug } }).pipe(Effect.ignore), |
| 127 | + ], |
| 128 | + { discard: true }, |
| 129 | + ), |
| 130 | + ); |
| 131 | + |
| 132 | + yield* client.openapi.addSpec({ |
| 133 | + payload: { |
| 134 | + spec: { kind: "blob", value: identitySpec(upstreamUrl, `Churn ${slug}`) }, |
| 135 | + slug, |
| 136 | + baseUrl: upstreamUrl, |
| 137 | + authenticationTemplate: [ |
| 138 | + { |
| 139 | + slug: "apiKey", |
| 140 | + type: "apiKey", |
| 141 | + headers: { authorization: ["Bearer ", { type: "variable", name: "token" }] }, |
| 142 | + }, |
| 143 | + ], |
| 144 | + }, |
| 145 | + }); |
| 146 | + |
| 147 | + if (options.probe) { |
| 148 | + const candidates = yield* client.integrations.healthCheckCandidates({ params: { slug } }); |
| 149 | + const getMe = candidates.find((candidate) => candidate.method === "get"); |
| 150 | + if (!getMe) return yield* Effect.die("identity spec exposed no GET candidate"); |
| 151 | + yield* client.integrations.healthCheckSet({ |
| 152 | + params: { slug }, |
| 153 | + payload: { spec: { operation: getMe.operation, identityField: "email" } }, |
| 154 | + }); |
| 155 | + } |
| 156 | + |
| 157 | + yield* client.connections.create({ |
| 158 | + payload: { owner: "org", name, integration: slug, template: TEMPLATE, value: "bad-key" }, |
| 159 | + }); |
| 160 | + |
| 161 | + return { slug, name }; |
| 162 | + }); |
| 163 | + |
| 164 | +scenario( |
| 165 | + "Health checks (UI) · many broken connections across many integrations settle instead of looping", |
| 166 | + {}, |
| 167 | + Effect.scoped( |
| 168 | + Effect.gen(function* () { |
| 169 | + const target = yield* Target; |
| 170 | + const browser = yield* Browser; |
| 171 | + const { client: makeClient } = yield* Api; |
| 172 | + const identity = yield* target.newIdentity(); |
| 173 | + const client = yield* makeClient(api, identity); |
| 174 | + const upstream = yield* serveRejectingUpstream(); |
| 175 | + |
| 176 | + const degraded = yield* Effect.all( |
| 177 | + Array.from({ length: DEGRADED_COUNT }, () => |
| 178 | + seedIntegration(client, upstream.url, { prefix: "churndeg", probe: true }), |
| 179 | + ), |
| 180 | + { concurrency: 2 }, |
| 181 | + ); |
| 182 | + const unknown = yield* Effect.all( |
| 183 | + Array.from({ length: UNKNOWN_COUNT }, () => |
| 184 | + seedIntegration(client, upstream.url, { prefix: "churnunk", probe: false }), |
| 185 | + ), |
| 186 | + { concurrency: 2 }, |
| 187 | + ); |
| 188 | + const seeded = [...degraded, ...unknown]; |
| 189 | + const connectionCount = seeded.length; |
| 190 | + |
| 191 | + // Persist a verdict on every connection BEFORE the browser opens, the |
| 192 | + // way a returning user finds them: `degraded` on the probed ones, |
| 193 | + // `unknown` on the rest. The list then renders each row with a |
| 194 | + // persisted verdict, and every automatic revalidation compares its |
| 195 | + // result against one. |
| 196 | + for (const { slug, name } of seeded) { |
| 197 | + yield* client.connections.checkHealth({ |
| 198 | + params: { owner: "org", integration: slug, name }, |
| 199 | + query: {}, |
| 200 | + }); |
| 201 | + } |
| 202 | + |
| 203 | + yield* browser.session(identity, async ({ page, step }) => { |
| 204 | + // The client's wire contract, observed from outside: every health POST |
| 205 | + // and every connections-list GET the app sends, plus how many the |
| 206 | + // browser reports as failed (an aborted request shows up here as a |
| 207 | + // failure, which is how the interrupted mutation atom looks on the wire). |
| 208 | + const health: string[] = []; |
| 209 | + const lists: string[] = []; |
| 210 | + let aborted = 0; |
| 211 | + // Only THIS scenario's connections count. Targets that share one org |
| 212 | + // across scenarios (selfhost) can carry rows another scenario left |
| 213 | + // behind, and those revalidate too; they are not ours to bound. |
| 214 | + const seededSlugs = new Set(seeded.map(({ slug }) => String(slug))); |
| 215 | + const isHealth = (method: string, url: string) => |
| 216 | + method === "POST" && |
| 217 | + url.includes("/health") && |
| 218 | + [...seededSlugs].some((slug) => url.includes(`/api/connections/org/${slug}/`)); |
| 219 | + const isList = (method: string, url: string) => |
| 220 | + method === "GET" && /\/api\/connections(\?|$)/.test(url); |
| 221 | + page.on("request", (request) => { |
| 222 | + const url = request.url(); |
| 223 | + if (isHealth(request.method(), url)) health.push(url); |
| 224 | + else if (isList(request.method(), url)) lists.push(url); |
| 225 | + }); |
| 226 | + page.on("requestfailed", (request) => { |
| 227 | + const url = request.url(); |
| 228 | + if (isHealth(request.method(), url) || isList(request.method(), url)) aborted += 1; |
| 229 | + }); |
| 230 | + |
| 231 | + const snapshot = (label: string) => { |
| 232 | + const line = `[churn] ${label}: health=${String(health.length)} lists=${String(lists.length)} aborted=${String(aborted)}`; |
| 233 | + console.log(line); |
| 234 | + return { health: health.length, lists: lists.length, aborted }; |
| 235 | + }; |
| 236 | + |
| 237 | + await step("Open the integrations list with every broken connection on it", async () => { |
| 238 | + await visit(page, "/"); |
| 239 | + for (const { slug } of seeded) { |
| 240 | + await page |
| 241 | + .getByRole("link", { name: new RegExp(slug, "i") }) |
| 242 | + .first() |
| 243 | + .waitFor({ |
| 244 | + timeout: 30_000, |
| 245 | + }); |
| 246 | + } |
| 247 | + // Let the automatic revalidation land for every row. |
| 248 | + await page.waitForTimeout(3_000); |
| 249 | + }); |
| 250 | + const afterList = snapshot("after integrations list"); |
| 251 | + |
| 252 | + await step("Open one integration page, then return to the list", async () => { |
| 253 | + await visit(page, `/integrations/${degraded[0]!.slug}`); |
| 254 | + await page.waitForTimeout(2_000); |
| 255 | + await visit(page, "/"); |
| 256 | + await page.waitForTimeout(3_000); |
| 257 | + }); |
| 258 | + const afterJourney = snapshot("after journey"); |
| 259 | + |
| 260 | + // The quiet window: nothing changed on the page, nothing changed on |
| 261 | + // the server, so nothing more should be asked. |
| 262 | + await step("Leave the settled list open and watch the wire", async () => { |
| 263 | + await page.waitForTimeout(QUIET_WINDOW_MS); |
| 264 | + }); |
| 265 | + const afterQuiet = snapshot("after quiet window"); |
| 266 | + |
| 267 | + const quietHealth = afterQuiet.health - afterJourney.health; |
| 268 | + const quietLists = afterQuiet.lists - afterJourney.lists; |
| 269 | + console.log( |
| 270 | + `[churn] quiet window (${String(QUIET_WINDOW_MS)}ms): +${String(quietHealth)} health, +${String(quietLists)} list fetches over ${String(connectionCount)} connections`, |
| 271 | + ); |
| 272 | + |
| 273 | + // One list mount revalidates each non-healthy connection once, and |
| 274 | + // reads the connections list once per owner (plus the unscoped read). |
| 275 | + // A probe count above the connection count means rows are being |
| 276 | + // re-probed; a list count above three means verdicts are being |
| 277 | + // treated as changes and refetching the list. |
| 278 | + expect( |
| 279 | + afterList.health, |
| 280 | + `the first list mount probes each broken connection at most once (sent ${String(afterList.health)} for ${String(connectionCount)} connections)`, |
| 281 | + ).toBeLessThanOrEqual(connectionCount); |
| 282 | + expect( |
| 283 | + afterList.lists, |
| 284 | + `the first list mount reads the connections list at most once per owner (sent ${String(afterList.lists)})`, |
| 285 | + ).toBeLessThanOrEqual(3); |
| 286 | + |
| 287 | + // THE production symptom: a settled page keeps asking. Zero is the |
| 288 | + // contract; anything else is the loop. |
| 289 | + expect(quietHealth, "a settled integrations list sends no further health probes").toBe(0); |
| 290 | + expect( |
| 291 | + quietLists, |
| 292 | + "a settled integrations list does not refetch the connections list", |
| 293 | + ).toBe(0); |
| 294 | + }); |
| 295 | + }), |
| 296 | + ), |
| 297 | +); |