History

Provision Autumn customers for orgs and self-heal customer_not_found

Rhys Sullivan committed 2 months ago

Commit 8f3317b1 parent 4c9e7555 files · +370 −18Browse files

Unified · Split

5 files
  1. apps/cloud/src/account/org-api-key-revoke.node.test.ts +1 −0
  2. apps/cloud/src/auth/handlers.ts +25 −2
  3. apps/cloud/src/extensions/billing/service.ts +95 −10
  4. e2e/cloud/billing-customer-provisioning.test.ts +205 −0
  5. e2e/src/surfaces/autumn.ts +44 −6
apps/cloud/src/account/org-api-key-revoke.node.test.ts+1−0
@@ -108,6 +108,7 @@ const stubUsers = Layer.succeed(UserStoreService)({ · 107 unmodified lines
108108
109109 const stubAutumn = Layer.succeed(AutumnService)({
110110 use: () => Effect.die("revoke does not touch billing"),
111+ ensureCustomer: () => Effect.die("revoke does not touch billing"),
111112 checkExecutionBalance: () => Effect.die("revoke does not touch billing"),
112113 trackExecution: () => Effect.void,
113114 });
apps/cloud/src/auth/handlers.ts+25−2
@@ -22,6 +22,7 @@ import { env } from "cloudflare:workers"; · 21 unmodified lines
2222 import { WorkOSError } from "./errors";
2323 import { WorkOSClient } from "./workos";
2424 import { AutumnService } from "../extensions/billing/service";
25+import { captureCauseEffect } from "../observability";
2526 import {
2627 hasPaidOrganizationSubscription,
2728 isOverFreeOrganizationLimit,
@@ -415,7 +416,9 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( · 387 unmodified lines
415416 ),
416417 { concurrency: 3 },
417418 ).pipe(
418− Effect.catchTag("AutumnError", () => Effect.fail(new WorkOSError())),
419+ // Any Autumn failure here (outage or missing customer) leaves the
420+ // paid/free split unknown, and the limit must fail closed.
421+ Effect.mapError(() => new WorkOSError()),
419422 Effect.map((ids) => new Set(ids.filter(Predicate.isNotNull))),
420423 );
421424
@@ -431,6 +434,24 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( · 9 unmodified lines
431434 s.upsertOrganization({ id: org.id, name: org.name }),
432435 );
433436
437+ // Provision the org's billing customer while we're the ones creating
438+ // the org. Without this the first billing call an org ever makes is a
439+ // non-creating one (balance check / usage track), which 404s and keeps
440+ // 404ing — unlimited unbilled executions. Non-fatal: a billing blip
441+ // must not block signup, and the billing seam heals a customer that
442+ // is still missing later.
443+ yield* autumn.ensureCustomer(org.id).pipe(
444+ Effect.catch((error) =>
445+ Effect.gen(function* () {
446+ yield* Effect.logWarning(
447+ "createOrganization: could not provision the Autumn customer",
448+ { organizationId: org.id, error },
449+ );
450+ yield* captureCauseEffect(error);
451+ }),
452+ ),
453+ );
454+
434455 // Try to attach the new org to the current session. This can fail
435456 // (or silently return a session still scoped to the old org) when
436457 // the caller's current session is stale — most commonly after the
@@ -515,7 +536,9 @@ export const CloudSessionAuthHandlers = HttpApiBuilder.group( · 78 unmodified lines
515536 yield* autumn
516537 .use((client) => client.customers.delete({ customerId: organizationId }))
517538 .pipe(
518− Effect.catchTag("AutumnError", (error) =>
539+ // Includes the "customer never existed" answer: nothing to cancel
540+ // is a fine outcome for a deleted org, and it is still worth a line.
541+ Effect.catch((error) =>
519542 Effect.logWarning("deleteOrganization: failed to delete Autumn customer", {
520543 organizationId,
521544 error,
apps/cloud/src/extensions/billing/service.ts+95−10
@@ -17,15 +17,61 @@ export class AutumnError extends Data.TaggedError("AutumnError")<{ · 16 unmodified lines
1717 cause?: unknown;
1818 }> {}
1919
20+/**
21+ * Autumn has no customer record for the organization. Split out from
22+ * `AutumnError` because it is not an outage and must not be treated like one:
23+ * an outage is transient and the right answer is to fail open and page, while
24+ * a missing customer is a PERMANENT provisioning gap — every subsequent
25+ * balance call 404s the same way, so the org runs unbilled and unmetered
26+ * forever. Callers that own an organization id repair it (see
27+ * `withProvisionedCustomer`); everything else still surfaces as `AutumnError`.
28+ */
29+export class AutumnCustomerNotFoundError extends Data.TaggedError("AutumnCustomerNotFoundError")<{
30+ message: string;
31+ cause?: unknown;
32+}> {}
33+
34+export type AutumnFailure = AutumnError | AutumnCustomerNotFoundError;
35+
36+// Autumn's own error code for "no such customer", carried in the JSON body of
37+// a 404 (`{"message":"Customer <id> not found","code":"customer_not_found"}`).
38+const CUSTOMER_NOT_FOUND_CODE = "customer_not_found";
39+
40+/**
41+ * True when `cause` is Autumn's "no such customer" answer. The autumn-js SDK
42+ * throws an `AutumnError` carrying the raw HTTP `statusCode` and `body`; match
43+ * on the code rather than the status alone so an unrelated 404 (a removed
44+ * endpoint, a proxy) is still reported as a genuine failure.
45+ */
46+const isCustomerNotFoundCause = (cause: unknown): boolean => {
47+ if (typeof cause !== "object" || cause === null) return false;
48+ const { statusCode, body } = cause as { readonly statusCode?: unknown; readonly body?: unknown };
49+ if (statusCode !== 404 || typeof body !== "string") return false;
50+ // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: classifying a third-party SDK's raw response body; a body that isn't JSON simply isn't this error
51+ try {
52+ // oxlint-disable-next-line executor/no-json-parse -- boundary: the autumn-js SDK hands back the response body as an unvalidated string
53+ const parsed: unknown = JSON.parse(body);
54+ if (typeof parsed !== "object" || parsed === null) return false;
55+ return (parsed as { readonly code?: unknown }).code === CUSTOMER_NOT_FOUND_CODE;
56+ } catch {
57+ return false;
58+ }
59+};
60+
2061 // ---------------------------------------------------------------------------
2162 // Service interface
2263 // ---------------------------------------------------------------------------
2364
2465 export type IAutumnService = Readonly<{
25− use: <A>(fn: (client: Autumn) => Promise<A>) => Effect.Effect<A, AutumnError, never>;
66+ use: <A>(fn: (client: Autumn) => Promise<A>) => Effect.Effect<A, AutumnFailure, never>;
67+ /**
68+ * Provision the organization's Autumn customer, creating it if Autumn has
69+ * never seen it. Idempotent — safe to call on every org creation.
70+ */
71+ ensureCustomer: (organizationId: string) => Effect.Effect<void, AutumnFailure, never>;
2672 checkExecutionBalance: (
2773 organizationId: string,
28− ) => Effect.Effect<{ readonly allowed: boolean }, AutumnError, never>;
74+ ) => Effect.Effect<{ readonly allowed: boolean }, AutumnFailure, never>;
2975 /**
3076 * Fire-and-forget-safe execution usage tracker. Errors are caught and
3177 * logged; the returned Effect never fails. Callers typically
@@ -48,6 +94,7 @@ const make = Effect.sync(() => { · 16 unmodified lines
4894 );
4995 return {
5096 use: () => notConfigured,
97+ ensureCustomer: () => notConfigured,
5198 checkExecutionBalance: () => notConfigured,
5299 trackExecution: () => Effect.void,
53100 } satisfies IAutumnService;
@@ -62,16 +109,53 @@ const make = Effect.sync(() => { · 8 unmodified lines
62109 const use = <A>(fn: (client: Autumn) => Promise<A>) =>
63110 Effect.tryPromise({
64111 try: () => fn(client),
65− catch: (cause) => new AutumnError({ message: "Autumn SDK request failed", cause }),
66− }).pipe(Effect.withSpan(`autumn.${fn.name ?? "use"}`));
112+ catch: (cause): AutumnFailure =>
113+ isCustomerNotFoundCause(cause)
114+ ? new AutumnCustomerNotFoundError({
115+ message: "Autumn has no customer for this organization",
116+ cause,
117+ })
118+ : new AutumnError({ message: "Autumn SDK request failed", cause }),
119+ // An inline arrow's `name` is "" — not nullish — so `??` left every
120+ // Autumn call tracing as the bare span `autumn.`.
121+ }).pipe(Effect.withSpan(`autumn.${fn.name || "use"}`));
122+
123+ const ensureCustomer = (organizationId: string) =>
124+ Effect.asVoid(use((c) => c.customers.getOrCreate({ customerId: organizationId })));
125+
126+ /**
127+ * Run `operation`; if Autumn answers "no such customer", provision the
128+ * organization's customer and run it ONCE more.
129+ *
130+ * This is the seam that closes the provisioning hole. Both billing paths use
131+ * non-creating endpoints, so an organization Autumn never learned about
132+ * 404s here forever: the balance gate fails open (correct for an outage,
133+ * catastrophic as a steady state) and every usage track is lost. Repairing
134+ * the customer makes the retry land the call — and a genuine Autumn outage
135+ * still fails with `AutumnError` and still pages, unretried.
136+ */
137+ const withProvisionedCustomer = <A>(
138+ organizationId: string,
139+ operation: Effect.Effect<A, AutumnFailure>,
140+ ) =>
141+ operation.pipe(
142+ Effect.catchTag("AutumnCustomerNotFoundError", () =>
143+ Effect.gen(function* () {
144+ yield* Effect.annotateCurrentSpan({ "autumn.customer.provisioned": true });
145+ yield* ensureCustomer(organizationId);
146+ return yield* operation;
147+ }),
148+ ),
149+ );
67150
68151 const trackExecution = (organizationId: string) =>
69152 Effect.gen(function* () {
70153 yield* Effect.annotateCurrentSpan({ "autumn.customer.id": organizationId });
71− yield* use((c) =>
72− c.track({ customerId: organizationId, featureId: "executions", value: 1 }),
154+ yield* withProvisionedCustomer(
155+ organizationId,
156+ use((c) => c.track({ customerId: organizationId, featureId: "executions", value: 1 })),
73157 ).pipe(
74− Effect.catchTag("AutumnError", (error) =>
158+ Effect.catch((error) =>
75159 Effect.gen(function* () {
76160 // Silent billing data loss is worth paging on: autumn.trackExecution
77161 // is fire-and-forget so the caller doesn't handle it themselves.
@@ -88,13 +172,14 @@ const make = Effect.sync(() => { · 10 unmodified lines
88172 const checkExecutionBalance = (organizationId: string) =>
89173 Effect.gen(function* () {
90174 yield* Effect.annotateCurrentSpan({ "autumn.customer.id": organizationId });
91− const check = yield* use((c) =>
92− c.check({ customerId: organizationId, featureId: "executions" }),
175+ const check = yield* withProvisionedCustomer(
176+ organizationId,
177+ use((c) => c.check({ customerId: organizationId, featureId: "executions" })),
93178 );
94179 return { allowed: check.allowed };
95180 }).pipe(Effect.withSpan("autumn.checkExecutionBalance"));
96181
97− return { use, checkExecutionBalance, trackExecution } satisfies IAutumnService;
182+ return { use, ensureCustomer, checkExecutionBalance, trackExecution } satisfies IAutumnService;
98183 });
99184
100185 export class AutumnService extends Context.Service<AutumnService, IAutumnService>()(
e2e/cloud/billing-customer-provisioning.test.ts+205−0
@@ -0,0 +1,205 @@
1+// Cloud-only (billing): an organization must EXIST as a customer at the billing
2+// provider, or every billing call for it answers `customer_not_found` forever.
3+//
4+// That state is invisible from the product side and catastrophic underneath it:
5+// the balance gate fails open (by design — a billing outage must not stop
6+// executions), and usage tracking is fire-and-forget, so the org runs unlimited
7+// executions and NONE of them reach the meter. The product looks perfectly
8+// healthy while every execution is unbilled and unmetered.
9+//
10+// Three guarantees are pinned here, all read from Autumn's own state rather than
11+// from the execution's response (which can never show any of them):
12+//
13+// 1. creating an organization provisions its billing customer up front,
14+// 2. if a customer is missing anyway, the billing seam heals it in place and
15+// the execution still lands on the meter, and
16+// 3. any OTHER billing failure is left alone — the repair is scoped to the
17+// "no such customer" answer, so a real provider failure is still reported
18+// rather than quietly retried away.
19+//
20+// The missing customer is produced with the emulator's fault injector — one
21+// 404 `customer_not_found` per billing endpoint — because the emulator, like
22+// real Autumn's SDK flow, otherwise auto-creates customers on contact.
23+import { expect } from "@effect/vitest";
24+import { Effect } from "effect";
25+
26+import { scenario } from "../src/scenario";
27+import { Autumn, Billing, Mcp, Target } from "../src/services";
28+import type { Identity } from "../src/target";
29+
30+const emailOf = (identity: Identity): string => identity.credentials?.email ?? identity.label;
31+
32+/** The org the bearer is scoped to — the Autumn customer id every billing call
33+ * is made against — read from the JWT's public claims. */
34+const orgIdOf = (bearer: string): string => {
35+ const claims = JSON.parse(Buffer.from(bearer.split(".")[1] ?? "", "base64url").toString()) as {
36+ readonly org_id?: string;
37+ };
38+ if (!claims.org_id) throw new Error("orgIdOf: bearer carries no org_id claim");
39+ return claims.org_id;
40+};
41+
42+/** A 404 `customer_not_found`, byte-for-byte the shape Autumn answers with for
43+ * an organization it has no customer record for. */
44+const CUSTOMER_NOT_FOUND = {
45+ status: 404,
46+ body: { message: "Customer not found", code: "customer_not_found" },
47+} as const;
48+
49+/** A 404 that is NOT a missing customer — a moved route, a proxy, a gateway.
50+ * Autumn carries a different code, and that code is the entire safety margin
51+ * between "provision and retry" and "quietly retry away a real failure". */
52+const UNRELATED_NOT_FOUND = {
53+ status: 404,
54+ body: { message: "Not Found", code: "not_found" },
55+} as const;
56+
57+scenario(
58+ "Billing · creating an organization provisions it as a billing customer",
59+ { timeout: 180_000 },
60+ Effect.gen(function* () {
61+ yield* Billing;
62+ const autumn = yield* Autumn;
63+ const target = yield* Target;
64+ const mcp = yield* Mcp;
65+
66+ // A brand-new user creating their first organization: none of the
67+ // opportunistic billing lookups (the over-the-free-limit check, the members
68+ // page's seat count, the rate limiter's paid-plan exemption) are reached on
69+ // this journey, so org creation is the only chance to provision.
70+ const identity = yield* target.newIdentity();
71+ const bearer = yield* mcp.mintBearer(emailOf(identity));
72+ const organizationId = orgIdOf(bearer);
73+
74+ const customerIds = yield* autumn.customerIds();
75+ expect(
76+ customerIds,
77+ "the new organization exists as a customer at the billing provider",
78+ ).toContain(organizationId);
79+ }),
80+);
81+
82+scenario(
83+ "Billing · a missing billing customer is healed in place and the execution is still metered",
84+ { timeout: 180_000 },
85+ Effect.gen(function* () {
86+ yield* Billing;
87+ const autumn = yield* Autumn;
88+ const target = yield* Target;
89+ const mcp = yield* Mcp;
90+
91+ const identity = yield* target.newIdentity();
92+ const bearer = yield* mcp.mintBearer(emailOf(identity));
93+ const customerId = orgIdOf(bearer);
94+
95+ const before = yield* autumn.usageEvents({ customerId, featureId: "executions" });
96+ expect(before.length, "a brand-new org starts with zero metered executions").toBe(0);
97+
98+ yield* Effect.gen(function* () {
99+ // One 404 each: the next balance check and the next usage track both
100+ // answer "this customer does not exist". A single `times: 1` fault means
101+ // a seam that heals and retries gets through, while one that gives up
102+ // loses the usage permanently — exactly the production failure.
103+ yield* autumn.armFault({
104+ match: { operationId: "balances.check" },
105+ response: CUSTOMER_NOT_FOUND,
106+ times: 1,
107+ });
108+ yield* autumn.armFault({
109+ match: { operationId: "balances.track" },
110+ response: CUSTOMER_NOT_FOUND,
111+ times: 1,
112+ });
113+
114+ const session = mcp.session(identity);
115+ const result = yield* session.call("execute", { code: "return 6 * 7;" });
116+
117+ // The gate still fails open: a billing problem never blocks a customer.
118+ expect(result.ok, "the execution runs despite the missing billing customer").toBe(true);
119+ expect(result.text, "it returns its value").toContain("42");
120+
121+ // Both billing calls really did reach Autumn and really were rejected —
122+ // without this the scenario could pass on a fault that never armed.
123+ const checks = yield* autumn.ledgerFor("balances.check");
124+ expect(
125+ checks.some((entry) => entry.faulted),
126+ "the balance check reached Autumn and was answered customer_not_found",
127+ ).toBe(true);
128+ const tracks = yield* autumn.ledgerFor("balances.track");
129+ expect(
130+ tracks.some((entry) => entry.faulted),
131+ "the usage track reached Autumn and was answered customer_not_found",
132+ ).toBe(true);
133+
134+ // The guarantee: the seam provisions the customer and retries, so the
135+ // execution lands on the meter. Before the fix this ledger stays empty
136+ // forever — the execution ran, and nobody was ever billed for it.
137+ const metered = yield* autumn.expectUsage({
138+ customerId,
139+ featureId: "executions",
140+ count: 1,
141+ });
142+ expect(metered.length, "the execution is metered exactly once").toBe(1);
143+ expect(metered[0]?.value, "it meters a single unit").toBe(1);
144+
145+ // Healing the customer is what made the retry possible, so the customer
146+ // must exist at the provider afterwards.
147+ const customerIds = yield* autumn.customerIds();
148+ expect(customerIds, "the organization has a billing customer afterwards").toContain(
149+ customerId,
150+ );
151+ }).pipe(Effect.ensuring(autumn.clearFaults().pipe(Effect.ignore)));
152+ }),
153+);
154+
155+scenario(
156+ "Billing · an unrelated billing failure is reported, not retried away",
157+ { timeout: 180_000 },
158+ Effect.gen(function* () {
159+ yield* Billing;
160+ const autumn = yield* Autumn;
161+ const target = yield* Target;
162+ const mcp = yield* Mcp;
163+
164+ const identity = yield* target.newIdentity();
165+ const bearer = yield* mcp.mintBearer(emailOf(identity));
166+ const customerId = orgIdOf(bearer);
167+
168+ yield* Effect.gen(function* () {
169+ // A 404 that is not "no such customer". If the seam treated every 404 as
170+ // a provisioning gap it would create a customer and retry, turning a real
171+ // provider failure into a silent success — and the alert never fires.
172+ yield* autumn.armFault({
173+ match: { operationId: "balances.track" },
174+ response: UNRELATED_NOT_FOUND,
175+ times: 1,
176+ });
177+
178+ const session = mcp.session(identity);
179+ const faulted = yield* session.call("execute", { code: "return 1 + 1;" });
180+ expect(faulted.ok, "the execution runs — billing never blocks a customer").toBe(true);
181+
182+ // A second, unfaulted execution. Its usage landing is the barrier: the
183+ // first execution's track (and any retry of it) is already done by the
184+ // time this one is on the meter, so the counts below are settled without
185+ // waiting on a clock.
186+ const clean = yield* session.call("execute", { code: "return 2 + 2;" });
187+ expect(clean.ok, "the follow-up execution runs too").toBe(true);
188+ yield* autumn.expectUsage({ customerId, featureId: "executions", count: 1 });
189+
190+ const tracks = (yield* autumn.ledgerFor("balances.track")).filter(
191+ (entry) => entry.customerId === customerId,
192+ );
193+ expect(
194+ tracks.filter((entry) => entry.faulted).length,
195+ "the first usage track was answered with the unrelated 404",
196+ ).toBe(1);
197+ // Two executions, two attempts: the failed one was reported and dropped,
198+ // not repaired and replayed.
199+ expect(tracks.length, "the rejected usage track is not retried").toBe(2);
200+
201+ const metered = yield* autumn.usageEvents({ customerId, featureId: "executions" });
202+ expect(metered.length, "only the unfaulted execution reaches the meter").toBe(1);
203+ }).pipe(Effect.ensuring(autumn.clearFaults().pipe(Effect.ignore)));
204+ }),
205+);
e2e/src/surfaces/autumn.ts+44−6
@@ -51,11 +51,19 @@ export interface LedgerEntry { · 50 unmodified lines
5151 readonly method: string;
5252 readonly path: string;
5353 readonly faulted: boolean;
54+ /** The customer the request was made against, read from its body. Autumn's
55+ * ledger is shared by every scenario in the run, so attempts have to be
56+ * attributed to one organization before they can be counted. */
57+ readonly customerId?: string;
5458 }
5559
5660 export interface AutumnSurface {
5761 /** One-shot read of the matching usage events from the ledger. */
5862 readonly usageEvents: (query: UsageQuery) => Effect.Effect<readonly UsageEvent[], unknown>;
63+ /** Every customer id Autumn currently holds (`customers.list`). An org that
64+ * was never provisioned as a billing customer is simply absent — the state
65+ * in which every balance call 404s `customer_not_found` forever. */
66+ readonly customerIds: () => Effect.Effect<readonly string[], unknown>;
5967 /** Poll until at least `count` matching events have landed. The track is
6068 * forked and the worker drains it on `waitUntil` shortly after the execution
6169 * returns, so arrival is eventually-consistent — polling IS the contract:
@@ -129,6 +137,26 @@ export const makeAutumnSurface = (autumnUrl: string): AutumnSurface => { · 67 unmodified lines
129137 }));
130138 });
131139
140+ const customerIds = () =>
141+ Effect.gen(function* () {
142+ const response = yield* Effect.promise(() =>
143+ fetch(`${autumnUrl}/v1/customers.list`, {
144+ method: "POST",
145+ headers: { "content-type": "application/json" },
146+ body: "{}",
147+ }),
148+ );
149+ if (!response.ok) {
150+ return yield* Effect.fail(
151+ `autumn customers.list responded ${response.status}: ${yield* Effect.promise(() => response.text())}`,
152+ );
153+ }
154+ const body = (yield* Effect.promise(() => response.json())) as {
155+ readonly list?: ReadonlyArray<{ readonly id?: string }>;
156+ };
157+ return (body.list ?? []).map((customer) => customer.id ?? "");
158+ });
159+
132160 const settleCheckout = (sessionId: string) =>
133161 Effect.gen(function* () {
134162 const response = yield* Effect.promise(() =>
@@ -232,20 +260,30 @@ export const makeAutumnSurface = (autumnUrl: string): AutumnSurface => { · 97 unmodified lines
232260 readonly method?: string;
233261 readonly path?: string;
234262 readonly faulted?: boolean;
263+ readonly request?: { readonly body?: unknown };
235264 }>;
236265 };
237266 return (body.entries ?? [])
238267 .filter((entry) => entry.operationId === operationId)
239− .map((entry) => ({
240− operationId: entry.operationId,
241− method: entry.method ?? "",
242− path: entry.path ?? "",
243− faulted: entry.faulted === true,
244− }));
268+ .map((entry) => {
269+ const requestBody = entry.request?.body;
270+ const customerId =
271+ typeof requestBody === "object" && requestBody !== null
272+ ? (requestBody as { readonly customer_id?: unknown }).customer_id
273+ : undefined;
274+ return {
275+ operationId: entry.operationId,
276+ method: entry.method ?? "",
277+ path: entry.path ?? "",
278+ faulted: entry.faulted === true,
279+ customerId: typeof customerId === "string" ? customerId : undefined,
280+ };
281+ });
245282 });
246283
247284 return {
248285 usageEvents,
286+ customerIds,
249287 settleCheckout,
250288 exhaustExecutions,
251289 attachPlan,