281 lines · 12.3 KB · TypeScript
RawDownload
// Cloud-only (billing): an organization must EXIST as a customer at the billing// provider, or every billing call for it answers `customer_not_found` forever.//// That state is invisible from the product side and catastrophic underneath it:// the balance gate fails open (by design — a billing outage must not stop// executions), and usage tracking is fire-and-forget, so the org runs unlimited// executions and NONE of them reach the meter. The product looks perfectly// healthy while every execution is unbilled and unmetered.//// Three guarantees are pinned here, all read from Autumn's own state rather than// from the execution's response (which can never show any of them):////   1. creating an organization provisions its billing customer up front,//   2. if a customer is missing anyway, the billing seam heals it in place and//      the execution still lands on the meter, and//   3. any OTHER billing failure is left alone — the repair is scoped to the//      "no such customer" answer, so a real provider failure is still reported//      rather than quietly retried away.//// The missing customer is produced with the emulator's fault injector — one// 404 `customer_not_found` per billing endpoint — because the emulator, like// real Autumn's SDK flow, otherwise auto-creates customers on contact.import { expect } from "@effect/vitest";import { Effect, Schedule } from "effect"; import { scenario } from "../src/scenario";import { Autumn, Billing, Mcp, Target } from "../src/services";import type { Identity } from "../src/target"; const emailOf = (identity: Identity): string => identity.credentials?.email ?? identity.label; /** The org the bearer is scoped to — the Autumn customer id every billing call *  is made against — read from the JWT's public claims. */const orgIdOf = (bearer: string): string => {  const claims = JSON.parse(Buffer.from(bearer.split(".")[1] ?? "", "base64url").toString()) as {    readonly org_id?: string;  };  if (!claims.org_id) throw new Error("orgIdOf: bearer carries no org_id claim");  return claims.org_id;}; /** A 404 `customer_not_found`, byte-for-byte the shape Autumn answers with for *  an organization it has no customer record for. */const CUSTOMER_NOT_FOUND = {  status: 404,  body: { message: "Customer not found", code: "customer_not_found" },} as const; /** A 404 that is NOT a missing customer — a moved route, a proxy, a gateway. *  Autumn carries a different code, and that code is the entire safety margin *  between "provision and retry" and "quietly retry away a real failure". */const UNRELATED_NOT_FOUND = {  status: 404,  body: { message: "Not Found", code: "not_found" },} as const; scenario(  "Billing · creating an organization provisions it as a billing customer",  { timeout: 180_000 },  Effect.gen(function* () {    yield* Billing;    const autumn = yield* Autumn;    const target = yield* Target;    const mcp = yield* Mcp;     // A brand-new user creating their first organization: none of the    // opportunistic billing lookups (the over-the-free-limit check, the members    // page's seat count, the rate limiter's paid-plan exemption) are reached on    // this journey, so org creation is the only chance to provision.    const identity = yield* target.newIdentity();    const bearer = yield* mcp.mintBearer(emailOf(identity));    const organizationId = orgIdOf(bearer);     const customerIds = yield* autumn.customerIds();    expect(      customerIds,      "the new organization exists as a customer at the billing provider",    ).toContain(organizationId);  }),); scenario(  "Billing · a missing billing customer is healed in place and the execution is still metered",  { timeout: 180_000 },  Effect.gen(function* () {    yield* Billing;    const autumn = yield* Autumn;    const target = yield* Target;    const mcp = yield* Mcp;     const identity = yield* target.newIdentity();    const bearer = yield* mcp.mintBearer(emailOf(identity));    const customerId = orgIdOf(bearer);     const before = yield* autumn.usageEvents({ customerId, featureId: "executions" });    expect(before.length, "a brand-new org starts with zero metered executions").toBe(0);     yield* Effect.gen(function* () {      // One 404 each: the next balance check and the next usage track both      // answer "this customer does not exist". A single `times: 1` fault means      // a seam that heals and retries gets through, while one that gives up      // loses the usage permanently — exactly the production failure.      yield* autumn.armFault({        match: { operationId: "balances.check" },        response: CUSTOMER_NOT_FOUND,        times: 1,      });      yield* autumn.armFault({        match: { operationId: "balances.track" },        response: CUSTOMER_NOT_FOUND,        times: 1,      });       const session = mcp.session(identity);      const result = yield* session.call("execute", { code: "return 6 * 7;" });       // The gate still fails open: a billing problem never blocks a customer.      expect(result.ok, "the execution runs despite the missing billing customer").toBe(true);      expect(result.text, "it returns its value").toContain("42");       // Both billing calls really did reach Autumn and really were rejected —      // without this the scenario could pass on a fault that never armed. The      // ledger is shared by the whole run, so attribute to THIS org: another      // scenario's faulted call must not stand in for this one's.      const checks = (yield* autumn.ledgerFor("balances.check")).filter(        (entry) => entry.customerId === customerId,      );      expect(        checks.some((entry) => entry.faulted),        "the balance check reached Autumn and was answered customer_not_found",      ).toBe(true);      const tracks = (yield* autumn.ledgerFor("balances.track")).filter(        (entry) => entry.customerId === customerId,      );      expect(        tracks.some((entry) => entry.faulted),        "the usage track reached Autumn and was answered customer_not_found",      ).toBe(true);       // The guarantee: the seam provisions the customer and retries, so the      // execution lands on the meter. Before the fix this ledger stays empty      // forever — the execution ran, and nobody was ever billed for it.      const metered = yield* autumn.expectUsage({        customerId,        featureId: "executions",        count: 1,      });      expect(metered.length, "the execution is metered exactly once").toBe(1);      expect(metered[0]?.value, "it meters a single unit").toBe(1);       // Healing the customer is what made the retry possible, so the customer      // must exist at the provider afterwards.      const customerIds = yield* autumn.customerIds();      expect(customerIds, "the organization has a billing customer afterwards").toContain(        customerId,      );    }).pipe(Effect.ensuring(autumn.clearFaults().pipe(Effect.ignore)));  }),); scenario(  "Billing · an unrelated billing failure is reported, not retried away",  { timeout: 180_000 },  Effect.gen(function* () {    yield* Billing;    const autumn = yield* Autumn;    const target = yield* Target;    const mcp = yield* Mcp;     const identity = yield* target.newIdentity();    const bearer = yield* mcp.mintBearer(emailOf(identity));    const customerId = orgIdOf(bearer);     yield* Effect.gen(function* () {      // A 404 that is not "no such customer". If the seam treated every 404 as      // a provisioning gap it would create a customer and retry, turning a real      // provider failure into a silent success — and the alert never fires.      yield* autumn.armFault({        match: { operationId: "balances.track" },        response: UNRELATED_NOT_FOUND,        times: 1,      });       const session = mcp.session(identity);      const faulted = yield* session.call("execute", { code: "return 1 + 1;" });      expect(faulted.ok, "the execution runs — billing never blocks a customer").toBe(true);       // A second, unfaulted execution. Its usage landing is the barrier: the      // first execution's track (and any retry of it) is already done by the      // time this one is on the meter, so the counts below are settled without      // waiting on a clock.      const clean = yield* session.call("execute", { code: "return 2 + 2;" });      expect(clean.ok, "the follow-up execution runs too").toBe(true);      yield* autumn.expectUsage({ customerId, featureId: "executions", count: 1 });       const tracks = (yield* autumn.ledgerFor("balances.track")).filter(        (entry) => entry.customerId === customerId,      );      expect(        tracks.filter((entry) => entry.faulted).length,        "the first usage track was answered with the unrelated 404",      ).toBe(1);      // Two executions, two attempts: the failed one was reported and dropped,      // not repaired and replayed.      expect(tracks.length, "the rejected usage track is not retried").toBe(2);       const metered = yield* autumn.usageEvents({ customerId, featureId: "executions" });      expect(metered.length, "only the unfaulted execution reaches the meter").toBe(1);    }).pipe(Effect.ensuring(autumn.clearFaults().pipe(Effect.ignore)));  }),); scenario(  "Billing · a customer that cannot be provisioned is given up on, not retried in a loop",  { timeout: 180_000 },  Effect.gen(function* () {    yield* Billing;    const autumn = yield* Autumn;    const target = yield* Target;    const mcp = yield* Mcp;     const identity = yield* target.newIdentity();    const bearer = yield* mcp.mintBearer(emailOf(identity));    const customerId = orgIdOf(bearer);     // Attempts against THIS org's meter, polled until the seam has settled on    // `atLeast` of them — the ledger is the only place the retry is visible.    const trackAttempts = (atLeast: number) =>      autumn.ledgerFor("balances.track").pipe(        Effect.map((entries) => entries.filter((entry) => entry.customerId === customerId)),        Effect.filterOrFail(          (entries) => entries.length >= atLeast,          (entries) => `only ${entries.length}/${atLeast} usage-track attempts for ${customerId}`,        ),        Effect.retry(Schedule.both(Schedule.spaced("250 millis"), Schedule.recurs(40))),      );     yield* Effect.gen(function* () {      // "No such customer" that does NOT go away when the customer is created —      // a provisioning gap the repair genuinely cannot close (a rejected id, a      // provider that never materializes the record). The seam must repair,      // retry ONCE, report, and stop: a repair loop here runs in a forked,      // untimed fibre, so an unbounded one hammers the provider forever for a      // single execution.      yield* autumn.armFault({        match: { operationId: "balances.track" },        response: CUSTOMER_NOT_FOUND,        times: 20,      });       const session = mcp.session(identity);      const stuck = yield* session.call("execute", { code: "return 7 * 6;" });      expect(stuck.ok, "the execution runs — billing never blocks a customer").toBe(true);      expect(stuck.text, "it returns its value").toContain("42");       // The original attempt plus the one post-repair retry.      yield* trackAttempts(2);       // Clearing the fault re-opens the meter, and the next execution's usage      // landing is the barrier: whatever the first execution was going to do to      // the ledger is finished by then, so the count below is settled without      // waiting on a clock.      yield* autumn.clearFaults();      const clean = yield* session.call("execute", { code: "return 2 + 2;" });      expect(clean.ok, "the follow-up execution runs too").toBe(true);      yield* autumn.expectUsage({ customerId, featureId: "executions", count: 1 });       const tracks = (yield* autumn.ledgerFor("balances.track")).filter(        (entry) => entry.customerId === customerId,      );      expect(        tracks.filter((entry) => entry.faulted).length,        "the unrepairable customer is attempted exactly twice: once, then once after the repair",      ).toBe(2);      expect(tracks.length, "and no further attempt beyond the next execution's own").toBe(3);       const metered = yield* autumn.usageEvents({ customerId, featureId: "executions" });      expect(metered.length, "only the execution Autumn accepted reaches the meter").toBe(1);    }).pipe(Effect.ensuring(autumn.clearFaults().pipe(Effect.ignore)));  }),);