The per-org hourly cap was sized for free-tier abuse but applied to every org regardless of plan, so a paying customer running a heavy workload was blocked mid-run. Paid orgs now skip the cap. The exemption is resolved only once the counter reports an org over the limit, so the common path is unchanged, and the result is cached per org. An unresolved exemption does not fail open: that would disable the backstop during exactly the billing outage it exists to cover, so a stale positive is reused when available and the cap otherwise applies. Blocks are logged, which nothing did before.
Unified · Split
| @@ -0,0 +1,171 @@ | |||
| 1 | import { describe, expect, it } from "@effect/vitest"; | ||
| 2 | import { Data, Effect } from "effect"; | ||
| 3 | |||
| 4 | import type { ExecutionEngine } from "@executor-js/execution"; | ||
| 5 | |||
| 6 | import { makeExecutionRateLimiter } from "./execution-rate-limit"; | ||
| 7 | import { RATE_LIMIT_BLOCKED_MESSAGE } from "./execution-limit-messages"; | ||
| 8 | |||
| 9 | const ORG = "org_test"; | ||
| 10 | |||
| 11 | /** Stands in for whatever the real lookups fail with (Autumn down, DO down). */ | ||
| 12 | class UpstreamDownError extends Data.TaggedError("UpstreamDownError")<{ | ||
| 13 | readonly which: string; | ||
| 14 | }> {} | ||
| 15 | |||
| 16 | // A stand-in engine: `execute` resolves to a marker, so a test tells an allowed | ||
| 17 | // execution (marker) from a blocked one (the gate's error result) by which of | ||
| 18 | // the two came back. A blocked decision never reaches the engine at all. | ||
| 19 | const engineStub: ExecutionEngine = { | ||
| 20 | execute: () => Effect.succeed({ result: "ran" }), | ||
| 21 | executeWithPause: () => Effect.succeed({ status: "completed", result: { result: "ran" } }), | ||
| 22 | resume: () => Effect.succeed(null), | ||
| 23 | getPausedExecution: () => Effect.succeed(null), | ||
| 24 | pausedExecutionCount: () => Effect.succeed(0), | ||
| 25 | hasPausedExecutions: () => Effect.succeed(false), | ||
| 26 | getDescription: Effect.succeed("stub"), | ||
| 27 | }; | ||
| 28 | |||
| 29 | /** Counter that hands out a caller-controlled sequence of counts. */ | ||
| 30 | const countingIncrement = (counts: ReadonlyArray<number>) => { | ||
| 31 | let calls = 0; | ||
| 32 | return () => Effect.succeed(counts[Math.min(calls++, counts.length - 1)] ?? 0); | ||
| 33 | }; | ||
| 34 | |||
| 35 | const runExecute = (limiter: ReturnType<typeof makeExecutionRateLimiter>) => | ||
| 36 | Effect.runPromise( | ||
| 37 | limiter.decorate(ORG, engineStub).execute("code", { | ||
| 38 | // Never invoked: the stub ignores it, and a blocked execution never runs. | ||
| 39 | onElicitation: () => Effect.die("elicitation is not exercised here"), | ||
| 40 | }), | ||
| 41 | ); | ||
| 42 | |||
| 43 | describe("execution rate limiter — paid exemption", () => { | ||
| 44 | it("allows executions under the cap without consulting the exemption", async () => { | ||
| 45 | let exemptionCalls = 0; | ||
| 46 | const limiter = makeExecutionRateLimiter(countingIncrement([1]), { | ||
| 47 | limit: 10, | ||
| 48 | isExempt: () => { | ||
| 49 | exemptionCalls += 1; | ||
| 50 | return Effect.succeed(false); | ||
| 51 | }, | ||
| 52 | }); | ||
| 53 | |||
| 54 | expect(await runExecute(limiter)).toMatchObject({ result: "ran" }); | ||
| 55 | // The whole point of resolving lazily: the common path costs no lookup. | ||
| 56 | expect(exemptionCalls).toBe(0); | ||
| 57 | }); | ||
| 58 | |||
| 59 | it("blocks a non-exempt org over the cap", async () => { | ||
| 60 | const limiter = makeExecutionRateLimiter(countingIncrement([11]), { | ||
| 61 | limit: 10, | ||
| 62 | isExempt: () => Effect.succeed(false), | ||
| 63 | }); | ||
| 64 | |||
| 65 | expect(await runExecute(limiter)).toMatchObject({ | ||
| 66 | result: null, | ||
| 67 | error: RATE_LIMIT_BLOCKED_MESSAGE, | ||
| 68 | }); | ||
| 69 | }); | ||
| 70 | |||
| 71 | it("allows an exempt org over the cap", async () => { | ||
| 72 | const limiter = makeExecutionRateLimiter(countingIncrement([11]), { | ||
| 73 | limit: 10, | ||
| 74 | isExempt: () => Effect.succeed(true), | ||
| 75 | }); | ||
| 76 | |||
| 77 | expect(await runExecute(limiter)).toMatchObject({ result: "ran" }); | ||
| 78 | }); | ||
| 79 | |||
| 80 | it("caches the exemption so a paid org past the cap looks it up once", async () => { | ||
| 81 | let exemptionCalls = 0; | ||
| 82 | const limiter = makeExecutionRateLimiter(countingIncrement([11, 12, 13]), { | ||
| 83 | limit: 10, | ||
| 84 | exemptionTtlMs: 60_000, | ||
| 85 | now: () => 1_000, | ||
| 86 | isExempt: () => { | ||
| 87 | exemptionCalls += 1; | ||
| 88 | return Effect.succeed(true); | ||
| 89 | }, | ||
| 90 | }); | ||
| 91 | |||
| 92 | for (let i = 0; i < 3; i += 1) | ||
| 93 | expect(await runExecute(limiter)).toMatchObject({ result: "ran" }); | ||
| 94 | expect(exemptionCalls).toBe(1); | ||
| 95 | }); | ||
| 96 | |||
| 97 | it("re-resolves once the cached exemption expires", async () => { | ||
| 98 | let exemptionCalls = 0; | ||
| 99 | let clock = 1_000; | ||
| 100 | const limiter = makeExecutionRateLimiter(countingIncrement([11, 12]), { | ||
| 101 | limit: 10, | ||
| 102 | exemptionTtlMs: 1_000, | ||
| 103 | now: () => clock, | ||
| 104 | isExempt: () => { | ||
| 105 | exemptionCalls += 1; | ||
| 106 | return Effect.succeed(true); | ||
| 107 | }, | ||
| 108 | }); | ||
| 109 | |||
| 110 | await runExecute(limiter); | ||
| 111 | clock += 5_000; | ||
| 112 | await runExecute(limiter); | ||
| 113 | expect(exemptionCalls).toBe(2); | ||
| 114 | }); | ||
| 115 | |||
| 116 | it("blocks when the exemption cannot be resolved and nothing is cached", async () => { | ||
| 117 | // Deliberately NOT fail-open: an unresolvable exemption during an Autumn | ||
| 118 | // outage must not switch the backstop off, which is the one scenario it | ||
| 119 | // exists to cover. | ||
| 120 | const limiter = makeExecutionRateLimiter(countingIncrement([11]), { | ||
| 121 | limit: 10, | ||
| 122 | isExempt: () => Effect.fail(new UpstreamDownError({ which: "autumn" })), | ||
| 123 | }); | ||
| 124 | |||
| 125 | expect(await runExecute(limiter)).toMatchObject({ | ||
| 126 | result: null, | ||
| 127 | error: RATE_LIMIT_BLOCKED_MESSAGE, | ||
| 128 | }); | ||
| 129 | }); | ||
| 130 | |||
| 131 | it("honours a stale exemption when a later lookup fails", async () => { | ||
| 132 | let clock = 1_000; | ||
| 133 | let shouldFail = false; | ||
| 134 | const limiter = makeExecutionRateLimiter(countingIncrement([11, 12]), { | ||
| 135 | limit: 10, | ||
| 136 | exemptionTtlMs: 1_000, | ||
| 137 | now: () => clock, | ||
| 138 | isExempt: () => | ||
| 139 | shouldFail ? Effect.fail(new UpstreamDownError({ which: "autumn" })) : Effect.succeed(true), | ||
| 140 | }); | ||
| 141 | |||
| 142 | expect(await runExecute(limiter)).toMatchObject({ result: "ran" }); | ||
| 143 | |||
| 144 | // Cache expires and Autumn is now unreachable: the known-paid org keeps | ||
| 145 | // running rather than getting blocked mid-workload by a blip. | ||
| 146 | clock += 5_000; | ||
| 147 | shouldFail = true; | ||
| 148 | expect(await runExecute(limiter)).toMatchObject({ result: "ran" }); | ||
| 149 | }); | ||
| 150 | |||
| 151 | it("fails open when the counter itself is unreachable", async () => { | ||
| 152 | const limiter = makeExecutionRateLimiter( | ||
| 153 | () => Effect.fail(new UpstreamDownError({ which: "counter DO" })), | ||
| 154 | { | ||
| 155 | limit: 10, | ||
| 156 | isExempt: () => Effect.succeed(false), | ||
| 157 | }, | ||
| 158 | ); | ||
| 159 | |||
| 160 | expect(await runExecute(limiter)).toMatchObject({ result: "ran" }); | ||
| 161 | }); | ||
| 162 | |||
| 163 | it("applies the cap when no exemption predicate is wired", async () => { | ||
| 164 | const limiter = makeExecutionRateLimiter(countingIncrement([11]), { limit: 10 }); | ||
| 165 | |||
| 166 | expect(await runExecute(limiter)).toMatchObject({ | ||
| 167 | result: null, | ||
| 168 | error: RATE_LIMIT_BLOCKED_MESSAGE, | ||
| 169 | }); | ||
| 170 | }); | ||
| 171 | }); | ||
| @@ -1,5 +1,5 @@ | |||
| 1 | // --------------------------------------------------------------------------- | 1 | // --------------------------------------------------------------------------- |
| 2 | // Per-org execution rate limit — an abuse backstop independent of billing. | 2 | // Per-org execution rate limit — a free-tier abuse backstop. |
| 3 | // | 3 | // |
| 4 | // The balance gate (execution-gate.ts) depends on Autumn and fails open, so a | 4 | // The balance gate (execution-gate.ts) depends on Autumn and fails open, so a |
| 5 | // billing outage plus runaway automation could still run unbounded executions. | 5 | // billing outage plus runaway automation could still run unbounded executions. |
| @@ -8,9 +8,22 @@ · 2 unmodified lines | |||
| 8 | // each MCP session lives in its own DO instance, so an in-memory counter | 8 | // each MCP session lives in its own DO instance, so an in-memory counter |
| 9 | // would be per-session and trivially bypassed by opening more sessions). | 9 | // would be per-session and trivially bypassed by opening more sessions). |
| 10 | // | 10 | // |
| 11 | // Like the balance gate it FAILS OPEN: an unreachable counter DO, a missing | 11 | // Paid organizations are EXEMPT. The cap was sized for free-tier abuse but |
| 12 | // applied to everyone, and on 2026-08-18 it blocked a paying customer | ||
| 13 | // mid-workload — their agent gave up on Executor and routed around it. Paid | ||
| 14 | // usage is what the balance gate and metered overage are for; this backstop | ||
| 15 | // has no business capping it. | ||
| 16 | // | ||
| 17 | // The exemption is resolved ONLY once the counter reports an org over the cap, | ||
| 18 | // so the common path (under the cap) costs the counter increment and nothing | ||
| 19 | // else. `isExempt` is an opaque predicate: this module still names no billing | ||
| 20 | // concept, and the Autumn coupling lives in `execution-stack-metered.ts`, | ||
| 21 | // which already owns that dependency. | ||
| 22 | // | ||
| 23 | // FAIL OPEN applies to the COUNTER: an unreachable counter DO, a missing | ||
| 12 | // binding, or a slow call allows the execution (warn + Sentry). The backstop | 24 | // binding, or a slow call allows the execution (warn + Sentry). The backstop |
| 13 | // must never take executions down with it. | 25 | // must never take executions down with it. An unresolved EXEMPTION is the one |
| 26 | // thing that does not fail open — see `resolveExemption`. | ||
| 14 | // --------------------------------------------------------------------------- | 27 | // --------------------------------------------------------------------------- |
| 15 | 28 | ||
| 16 | import { DurableObject, env } from "cloudflare:workers"; | 29 | import { DurableObject, env } from "cloudflare:workers"; |
| @@ -25,14 +38,26 @@ import { RATE_LIMIT_BLOCKED_MESSAGE } from "./execution-limit-messages"; · 8 unmodified lines | |||
| 25 | 38 | ||
| 26 | // Fixed window: all executions in the same clock hour share one counter. | 39 | // Fixed window: all executions in the same clock hour share one counter. |
| 27 | export const RATE_LIMIT_WINDOW_MS = 3_600_000; | 40 | export const RATE_LIMIT_WINDOW_MS = 3_600_000; |
| 28 | // Calibration: the heaviest legitimate org runs ~1.1k executions per MONTH, | 41 | // The cap for organizations WITHOUT a paid subscription. |
| 29 | // so 1000 per HOUR is far above any human-driven usage and only trips on | 42 | // |
| 30 | // runaway automation (the incident this backstops: ~18k in 30 days would | 43 | // The original calibration ("the heaviest legitimate org runs ~1.1k executions |
| 31 | // still pass, which is fine — that class of overrun is the balance gate's | 44 | // per MONTH, so 1000 per HOUR is far above any human-driven usage") went stale |
| 32 | // job; this catches tight loops). | 45 | // inside six weeks: by 2026-08-18 a paying org was sustaining ~5.3k executions |
| 46 | // per DAY and crossed this cap in a single hour. Sizing a shared number | ||
| 47 | // against the largest customer is a losing game, so the number no longer tries | ||
| 48 | // to describe them — paid orgs are exempt below, and this now has only | ||
| 49 | // free-tier abuse to cover, which is what it was picked for. | ||
| 33 | export const EXECUTIONS_PER_ORG_PER_HOUR = 1000; | 50 | export const EXECUTIONS_PER_ORG_PER_HOUR = 1000; |
| 34 | // Counter DO slower than this => fail open rather than stall executions. | 51 | // Counter DO slower than this => fail open rather than stall executions. |
| 35 | const RATE_LIMIT_CHECK_TIMEOUT_MS = 2_000; | 52 | const RATE_LIMIT_CHECK_TIMEOUT_MS = 2_000; |
| 53 | // Exemption lookup slower than this => treat as unresolved. | ||
| 54 | const EXEMPTION_CHECK_TIMEOUT_MS = 2_000; | ||
| 55 | // An org over the cap is checked at most once per TTL rather than once per | ||
| 56 | // execution, so a paid org running far past it doesn't hammer the lookup. | ||
| 57 | const EXEMPTION_CACHE_TTL_MS = 60_000; | ||
| 58 | // Sweep guard, mirroring the balance gate's cache: one long-lived isolate can | ||
| 59 | // serve many orgs. | ||
| 60 | const EXEMPTION_CACHE_MAX_ENTRIES = 10_000; | ||
| 36 | // The DO purges its storage this long after the last increment, so idle orgs | 61 | // The DO purges its storage this long after the last increment, so idle orgs |
| 37 | // cost nothing. Two windows: long enough that an active window never purges. | 62 | // cost nothing. Two windows: long enough that an active window never purges. |
| 38 | const COUNTER_PURGE_AFTER_MS = 2 * RATE_LIMIT_WINDOW_MS; | 63 | const COUNTER_PURGE_AFTER_MS = 2 * RATE_LIMIT_WINDOW_MS; |
| @@ -51,6 +76,20 @@ class RateLimitCheckTimeoutError extends Data.TaggedError("RateLimitCheckTimeout · 12 unmodified lines | |||
| 51 | readonly timeoutMs: number; | 76 | readonly timeoutMs: number; |
| 52 | }> {} | 77 | }> {} |
| 53 | 78 | ||
| 79 | /** Internal sentinel for an exemption lookup that exceeded its time budget. */ | ||
| 80 | class ExemptionCheckTimeoutError extends Data.TaggedError("ExemptionCheckTimeoutError")<{ | ||
| 81 | readonly timeoutMs: number; | ||
| 82 | }> {} | ||
| 83 | |||
| 84 | /** | ||
| 85 | * Whether an organization is exempt from the cap. Production passes a paid- | ||
| 86 | * subscription check; keeping it an opaque predicate is what lets this module | ||
| 87 | * stay free of any billing import. | ||
| 88 | */ | ||
| 89 | export type ExecutionRateLimitExemption = ( | ||
| 90 | organizationId: string, | ||
| 91 | ) => Effect.Effect<boolean, unknown>; | ||
| 92 | |||
| 54 | // --------------------------------------------------------------------------- | 93 | // --------------------------------------------------------------------------- |
| 55 | // Counter Durable Object — one instance per organization (idFromName(orgId)). | 94 | // Counter Durable Object — one instance per organization (idFromName(orgId)). |
| 56 | // Stores a single { windowId, count } record: an increment in a new window | 95 | // Stores a single { windowId, count } record: an increment in a new window |
| @@ -119,12 +158,76 @@ export const makeExecutionRateLimiter = ( · 62 unmodified lines | |||
| 119 | readonly windowMs?: number; | 158 | readonly windowMs?: number; |
| 120 | readonly timeoutMs?: number; | 159 | readonly timeoutMs?: number; |
| 121 | readonly now?: () => number; | 160 | readonly now?: () => number; |
| 161 | readonly isExempt?: ExecutionRateLimitExemption; | ||
| 162 | readonly exemptionTtlMs?: number; | ||
| 122 | }, | 163 | }, |
| 123 | ): ExecutionRateLimiter => { | 164 | ): ExecutionRateLimiter => { |
| 124 | const limit = options?.limit ?? EXECUTIONS_PER_ORG_PER_HOUR; | 165 | const limit = options?.limit ?? EXECUTIONS_PER_ORG_PER_HOUR; |
| 125 | const windowMs = options?.windowMs ?? RATE_LIMIT_WINDOW_MS; | 166 | const windowMs = options?.windowMs ?? RATE_LIMIT_WINDOW_MS; |
| 126 | const timeoutMs = options?.timeoutMs ?? RATE_LIMIT_CHECK_TIMEOUT_MS; | 167 | const timeoutMs = options?.timeoutMs ?? RATE_LIMIT_CHECK_TIMEOUT_MS; |
| 127 | const now = options?.now ?? Date.now; | 168 | const now = options?.now ?? Date.now; |
| 169 | const isExempt = options?.isExempt; | ||
| 170 | const exemptionTtlMs = options?.exemptionTtlMs ?? EXEMPTION_CACHE_TTL_MS; | ||
| 171 | |||
| 172 | const exemptionCache = new Map< | ||
| 173 | string, | ||
| 174 | { readonly exempt: boolean; readonly expiresAtMs: number } | ||
| 175 | >([]); | ||
| 176 | |||
| 177 | const writeExemptionCache = (organizationId: string, exempt: boolean, nowMs: number): void => { | ||
| 178 | if (exemptionCache.size >= EXEMPTION_CACHE_MAX_ENTRIES) { | ||
| 179 | for (const [key, entry] of exemptionCache) { | ||
| 180 | if (entry.expiresAtMs <= nowMs) exemptionCache.delete(key); | ||
| 181 | } | ||
| 182 | // Still saturated after dropping expired entries: reset rather than grow. | ||
| 183 | if (exemptionCache.size >= EXEMPTION_CACHE_MAX_ENTRIES) exemptionCache.clear(); | ||
| 184 | } | ||
| 185 | exemptionCache.set(organizationId, { exempt, expiresAtMs: nowMs + exemptionTtlMs }); | ||
| 186 | }; | ||
| 187 | |||
| 188 | /** | ||
| 189 | * Resolved only for orgs already over the cap, so the lookup never touches | ||
| 190 | * the common path. | ||
| 191 | * | ||
| 192 | * This is the one place that does NOT fail open. The balance gate already | ||
| 193 | * allows executions when Autumn is unreachable; if the exemption did too, | ||
| 194 | * an Autumn outage would switch this backstop off entirely — precisely the | ||
| 195 | * "billing outage plus runaway automation" case it exists to cover. A stale | ||
| 196 | * positive is honoured ahead of that fallback, so a blip can't flip a | ||
| 197 | * known-paid org into a block mid-workload. | ||
| 198 | */ | ||
| 199 | const resolveExemption = (organizationId: string): Effect.Effect<boolean> => | ||
| 200 | Effect.suspend(() => { | ||
| 201 | if (!isExempt) return Effect.succeed(false); | ||
| 202 | const nowMs = now(); | ||
| 203 | const cached = exemptionCache.get(organizationId); | ||
| 204 | if (cached && cached.expiresAtMs > nowMs) return Effect.succeed(cached.exempt); | ||
| 205 | return isExempt(organizationId).pipe( | ||
| 206 | Effect.timeoutOrElse({ | ||
| 207 | duration: `${EXEMPTION_CHECK_TIMEOUT_MS} millis`, | ||
| 208 | orElse: () => | ||
| 209 | Effect.fail(new ExemptionCheckTimeoutError({ timeoutMs: EXEMPTION_CHECK_TIMEOUT_MS })), | ||
| 210 | }), | ||
| 211 | Effect.map((exempt) => { | ||
| 212 | writeExemptionCache(organizationId, exempt, nowMs); | ||
| 213 | return exempt; | ||
| 214 | }), | ||
| 215 | Effect.catch((error: unknown) => | ||
| 216 | Effect.gen(function* () { | ||
| 217 | yield* Effect.sync(() => { | ||
| 218 | console.warn( | ||
| 219 | `[rate-limit] exemption lookup failed for ${organizationId}; treating as ${ | ||
| 220 | cached ? "last known" : "not exempt" | ||
| 221 | }:`, | ||
| 222 | error, | ||
| 223 | ); | ||
| 224 | }); | ||
| 225 | yield* captureCauseEffect(error); | ||
| 226 | return cached?.exempt ?? false; | ||
| 227 | }), | ||
| 228 | ), | ||
| 229 | ); | ||
| 230 | }); | ||
| 128 | 231 | ||
| 129 | const decide = (organizationId: string): Effect.Effect<GateDecision> => | 232 | const decide = (organizationId: string): Effect.Effect<GateDecision> => |
| 130 | Effect.suspend(() => { | 233 | Effect.suspend(() => { |
| @@ -134,18 +237,34 @@ export const makeExecutionRateLimiter = ( · 3 unmodified lines | |||
| 134 | duration: `${timeoutMs} millis`, | 237 | duration: `${timeoutMs} millis`, |
| 135 | orElse: () => Effect.fail(new RateLimitCheckTimeoutError({ timeoutMs })), | 238 | orElse: () => Effect.fail(new RateLimitCheckTimeoutError({ timeoutMs })), |
| 136 | }), | 239 | }), |
| 137 | Effect.map( | 240 | Effect.flatMap((count): Effect.Effect<GateDecision> => { |
| 138 | (count): GateDecision => | 241 | // Under the cap: no exemption lookup, no extra I/O. |
| 139 | count > limit | 242 | if (count <= limit) return Effect.succeed({ blocked: false }); |
| 140 | ? { | 243 | return Effect.gen(function* () { |
| 141 | blocked: true, | 244 | if (yield* resolveExemption(organizationId)) { |
| 142 | error: new ExecutionRateLimitExceededError({ | 245 | return { blocked: false } as const satisfies GateDecision; |
| 143 | organizationId, | 246 | } |
| 144 | message: RATE_LIMIT_BLOCKED_MESSAGE, | 247 | // The only record that the backstop fired. A blocked execution is |
| 145 | }), | 248 | // never usage-tracked (the gate short-circuits before the tracker) |
| 146 | } | 249 | // and deliberately not sent to Sentry — a backstop stopping |
| 147 | : { blocked: false }, | 250 | // runaway automation is expected, not exceptional — so without |
| 148 | ), | 251 | // this line a blocked org is invisible outside a bug report, which |
| 252 | // is how the 2026-08-18 block went unnoticed until a customer | ||
| 253 | // sent a screenshot. | ||
| 254 | yield* Effect.sync(() => { | ||
| 255 | console.warn( | ||
| 256 | `[rate-limit] blocked execution for ${organizationId}: ${count} > ${limit} in window ${windowId}`, | ||
| 257 | ); | ||
| 258 | }); | ||
| 259 | return { | ||
| 260 | blocked: true, | ||
| 261 | error: new ExecutionRateLimitExceededError({ | ||
| 262 | organizationId, | ||
| 263 | message: RATE_LIMIT_BLOCKED_MESSAGE, | ||
| 264 | }), | ||
| 265 | } as const satisfies GateDecision; | ||
| 266 | }); | ||
| 267 | }), | ||
| 149 | // FAIL OPEN: the backstop must never block executions because its | 268 | // FAIL OPEN: the backstop must never block executions because its |
| 150 | // counter is unreachable or slow. | 269 | // counter is unreachable or slow. |
| 151 | Effect.catch((error: unknown) => | 270 | Effect.catch((error: unknown) => |
| @@ -185,8 +304,13 @@ type RateLimiterNamespace = { · 33 unmodified lines | |||
| 185 | * Production rate limiter backed by the `EXECUTION_RATE_LIMITER` counter DO. | 304 | * Production rate limiter backed by the `EXECUTION_RATE_LIMITER` counter DO. |
| 186 | * When the binding is absent (unit-test workers, older local setups) the | 305 | * When the binding is absent (unit-test workers, older local setups) the |
| 187 | * limiter is disabled: every check passes, logged once at construction. | 306 | * limiter is disabled: every check passes, logged once at construction. |
| 307 | * | ||
| 308 | * `isExempt` decides which orgs the cap skips; production passes a paid- | ||
| 309 | * subscription check from `execution-stack-metered.ts`. | ||
| 188 | */ | 310 | */ |
| 189 | export const makeCloudExecutionRateLimiter = (): ExecutionRateLimiter => { | 311 | export const makeCloudExecutionRateLimiter = ( |
| 312 | isExempt: ExecutionRateLimitExemption, | ||
| 313 | ): ExecutionRateLimiter => { | ||
| 190 | const limit = resolveRateLimit(); | 314 | const limit = resolveRateLimit(); |
| 191 | const namespace = (env as { EXECUTION_RATE_LIMITER?: RateLimiterNamespace }) | 315 | const namespace = (env as { EXECUTION_RATE_LIMITER?: RateLimiterNamespace }) |
| 192 | .EXECUTION_RATE_LIMITER; | 316 | .EXECUTION_RATE_LIMITER; |
| @@ -204,7 +328,7 @@ export const makeCloudExecutionRateLimiter = (): ExecutionRateLimiter => { · 11 unmodified lines | |||
| 204 | ) as ExecutionRateLimiterStub; | 328 | ) as ExecutionRateLimiterStub; |
| 205 | return stub.increment(windowId); | 329 | return stub.increment(windowId); |
| 206 | }), | 330 | }), |
| 207 | { limit }, | 331 | { limit, isExempt }, |
| 208 | ); | 332 | ); |
| 209 | }; | 333 | }; |
| 210 | 334 | ||
| @@ -26,6 +26,7 @@ import { · 25 unmodified lines | |||
| 26 | } from "@executor-js/api/server"; | 26 | } from "@executor-js/api/server"; |
| 27 | 27 | ||
| 28 | import { AutumnService } from "../extensions/billing/service"; | 28 | import { AutumnService } from "../extensions/billing/service"; |
| 29 | import { hasPaidOrganizationSubscription } from "../extensions/billing/plans"; | ||
| 29 | import type { DbService } from "../db/db"; | 30 | import type { DbService } from "../db/db"; |
| 30 | import { CloudExecutionSeamsLayer } from "../engine/execution-stack"; | 31 | import { CloudExecutionSeamsLayer } from "../engine/execution-stack"; |
| 31 | import { makeExecutionLimitGate } from "./execution-gate"; | 32 | import { makeExecutionLimitGate } from "./execution-gate"; |
| @@ -34,7 +35,8 @@ import { withExecutionUsageTracking } from "./execution-usage"; · 2 unmodified lines | |||
| 34 | 35 | ||
| 35 | // Usage-metering decorator bound to the billing service, plus the two | 36 | // Usage-metering decorator bound to the billing service, plus the two |
| 36 | // pre-execution guards this layer owns, ordered cheapest first: | 37 | // pre-execution guards this layer owns, ordered cheapest first: |
| 37 | // 1. rate-limit backstop (counter DO, independent of billing) | 38 | // 1. rate-limit backstop (counter DO; free-tier abuse only — paid orgs are |
| 39 | // exempt via the subscription lookup below) | ||
| 38 | // 2. execution balance gate (Autumn check, cached 60s, fails open) | 40 | // 2. execution balance gate (Autumn check, cached 60s, fails open) |
| 39 | // 3. usage tracking — fire-and-forget (`Effect.runFork`) so the billing | 41 | // 3. usage tracking — fire-and-forget (`Effect.runFork`) so the billing |
| 40 | // call can't stall a user-facing execution. | 42 | // call can't stall a user-facing execution. |
| @@ -48,7 +50,18 @@ export const CloudMeteringEngineDecorator: Layer.Layer<EngineDecorator, never, A · 7 unmodified lines | |||
| 48 | const balanceGate = makeExecutionLimitGate((organizationId) => | 50 | const balanceGate = makeExecutionLimitGate((organizationId) => |
| 49 | autumn.checkExecutionBalance(organizationId), | 51 | autumn.checkExecutionBalance(organizationId), |
| 50 | ); | 52 | ); |
| 51 | const rateLimiter = makeCloudExecutionRateLimiter(); | 53 | // The limiter's paid-org exemption. This is the billing coupling the |
| 54 | // limiter module deliberately avoids owning, and it reads the same | ||
| 55 | // `PAID_AUTUMN_PLAN_IDS` config as the org-creation and seat gates so | ||
| 56 | // "paid" means one thing across the app. The limiter calls this only | ||
| 57 | // for orgs already over the cap and caches the answer, so the extra | ||
| 58 | // Autumn round trip stays off the hot path. | ||
| 59 | const rateLimiter = makeCloudExecutionRateLimiter((organizationId) => | ||
| 60 | Effect.map( | ||
| 61 | autumn.use((client) => client.customers.getOrCreate({ customerId: organizationId })), | ||
| 62 | (customer) => hasPaidOrganizationSubscription(customer.subscriptions), | ||
| 63 | ), | ||
| 64 | ); | ||
| 52 | return { | 65 | return { |
| 53 | decorate: (engine, identity: EngineStackIdentity) => | 66 | decorate: (engine, identity: EngineStackIdentity) => |
| 54 | rateLimiter.decorate( | 67 | rateLimiter.decorate( |
| @@ -198,3 +198,52 @@ scenario( · 197 unmodified lines | |||
| 198 | ); | 198 | ); |
| 199 | }), | 199 | }), |
| 200 | ); | 200 | ); |
| 201 | |||
| 202 | scenario( | ||
| 203 | "Billing · a paid org runs past the rate-limit backstop instead of being blocked", | ||
| 204 | { timeout: 180_000 }, | ||
| 205 | Effect.gen(function* () { | ||
| 206 | // The backstop is sized for free-tier abuse, and it used to apply to every | ||
| 207 | // org regardless of plan: on 2026-08-18 it blocked a paying customer | ||
| 208 | // mid-workload. This pins the exemption end to end — the same run that | ||
| 209 | // blocks a free org (the scenario above) must sail past the cap here. | ||
| 210 | yield* Billing; | ||
| 211 | const autumn = yield* Autumn; | ||
| 212 | const target = yield* Target; | ||
| 213 | const mcp = yield* Mcp; | ||
| 214 | |||
| 215 | const identity = yield* target.newIdentity(); | ||
| 216 | const bearer = yield* mcp.mintBearer(emailOf(identity)); | ||
| 217 | const customerId = orgIdOf(bearer); | ||
| 218 | |||
| 219 | // Enterprise, not Team: it carries no price and no card-required trial, so | ||
| 220 | // the emulator activates it inline rather than answering with a checkout | ||
| 221 | // URL. Its executions are unlimited, so the BALANCE gate can never be what | ||
| 222 | // allows or blocks here — the rate-limit backstop is isolated as the only | ||
| 223 | // guard under test. | ||
| 224 | yield* autumn.attachPlan(customerId, "enterprise"); | ||
| 225 | |||
| 226 | const session = mcp.session(identity); | ||
| 227 | |||
| 228 | // Deliberately past E2E_EXECUTION_RATE_LIMIT: the count that blocks a free | ||
| 229 | // org. Every one must run. | ||
| 230 | const overCap = RATE_LIMIT + 3; | ||
| 231 | for (let i = 1; i <= overCap; i++) { | ||
| 232 | const ok = yield* session.call("execute", { code: `return ${i};` }); | ||
| 233 | expect(ok.ok, `execution ${i} succeeds for a paid org (cap is ${RATE_LIMIT})`).toBe(true); | ||
| 234 | expect( | ||
| 235 | ok.text, | ||
| 236 | `execution ${i} returns its value rather than the backstop message`, | ||
| 237 | ).not.toContain(RATE_LIMIT_BLOCKED_MESSAGE); | ||
| 238 | } | ||
| 239 | |||
| 240 | // Nothing was withheld from the meter either: a paid org past the cap is | ||
| 241 | // billed for every execution it ran. | ||
| 242 | const metered = yield* autumn.expectUsage({ | ||
| 243 | customerId, | ||
| 244 | featureId: "executions", | ||
| 245 | count: overCap, | ||
| 246 | }); | ||
| 247 | expect(metered.length, "every execution past the cap is still metered").toBe(overCap); | ||
| 248 | }), | ||
| 249 | ); | ||
| @@ -76,6 +76,14 @@ export interface AutumnSurface { · 75 unmodified lines | |||
| 76 | * which drives `remaining` to zero. This is the setup a "blocked at cap" | 76 | * which drives `remaining` to zero. This is the setup a "blocked at cap" |
| 77 | * scenario runs before opening the session it expects to be gated. */ | 77 | * scenario runs before opening the session it expects to be gated. */ |
| 78 | readonly exhaustExecutions: (customerId: string) => Effect.Effect<void, unknown>; | 78 | readonly exhaustExecutions: (customerId: string) => Effect.Effect<void, unknown>; |
| 79 | /** Put an org on a plan directly (`billing.attach`), skipping the hosted | ||
| 80 | * checkout. Only valid for a plan that needs no payment — Enterprise carries | ||
| 81 | * no price and no card-required trial, so the emulator activates its | ||
| 82 | * subscription inline. Team does have both, so it must still go through the | ||
| 83 | * browser checkout + `settleCheckout`. This is the setup a scenario runs when | ||
| 84 | * it needs a PAID org and the upgrade journey itself is not what's under | ||
| 85 | * test (e.g. the rate-limit backstop's paid exemption). */ | ||
| 86 | readonly attachPlan: (customerId: string, planId: string) => Effect.Effect<void, unknown>; | ||
| 79 | /** Arm a fault against a matching Autumn request (`POST /_emulate/faults`). */ | 87 | /** Arm a fault against a matching Autumn request (`POST /_emulate/faults`). */ |
| 80 | readonly armFault: (input: FaultInput) => Effect.Effect<void, unknown>; | 88 | readonly armFault: (input: FaultInput) => Effect.Effect<void, unknown>; |
| 81 | /** Clear every armed fault (`DELETE /_emulate/faults`) — the finalizer that | 89 | /** Clear every armed fault (`DELETE /_emulate/faults`) — the finalizer that |
| @@ -155,6 +163,33 @@ export const makeAutumnSurface = (autumnUrl: string): AutumnSurface => { · 73 unmodified lines | |||
| 155 | } | 163 | } |
| 156 | }); | 164 | }); |
| 157 | 165 | ||
| 166 | const attachPlan = (customerId: string, planId: string) => | ||
| 167 | Effect.gen(function* () { | ||
| 168 | const response = yield* Effect.promise(() => | ||
| 169 | fetch(`${autumnUrl}/v1/billing.attach`, { | ||
| 170 | method: "POST", | ||
| 171 | headers: { "content-type": "application/json" }, | ||
| 172 | body: JSON.stringify({ customer_id: customerId, plan_id: planId }), | ||
| 173 | }), | ||
| 174 | ); | ||
| 175 | if (!response.ok) { | ||
| 176 | return yield* Effect.fail( | ||
| 177 | `autumn billing.attach responded ${response.status}: ${yield* Effect.promise(() => response.text())}`, | ||
| 178 | ); | ||
| 179 | } | ||
| 180 | // A plan that needs payment answers with a checkout URL instead of | ||
| 181 | // activating; that silently leaves the org on Free, and a scenario that | ||
| 182 | // asked for a paid org would then assert against the wrong plan. | ||
| 183 | const body = (yield* Effect.promise(() => response.json())) as { | ||
| 184 | readonly payment_url?: string | null; | ||
| 185 | }; | ||
| 186 | if (body.payment_url) { | ||
| 187 | return yield* Effect.fail( | ||
| 188 | `autumn billing.attach returned a checkout URL for '${planId}': it requires payment, so it cannot be attached directly`, | ||
| 189 | ); | ||
| 190 | } | ||
| 191 | }); | ||
| 192 | |||
| 158 | const armFault = (input: FaultInput) => | 193 | const armFault = (input: FaultInput) => |
| 159 | Effect.gen(function* () { | 194 | Effect.gen(function* () { |
| 160 | const response = yield* Effect.promise(() => | 195 | const response = yield* Effect.promise(() => |
| @@ -213,6 +248,7 @@ export const makeAutumnSurface = (autumnUrl: string): AutumnSurface => { · 52 unmodified lines | |||
| 213 | usageEvents, | 248 | usageEvents, |
| 214 | settleCheckout, | 249 | settleCheckout, |
| 215 | exhaustExecutions, | 250 | exhaustExecutions, |
| 251 | attachPlan, | ||
| 216 | armFault, | 252 | armFault, |
| 217 | clearFaults, | 253 | clearFaults, |
| 218 | ledgerFor, | 254 | ledgerFor, |