History

Add the pure tool-catalog sync lifecycle model

Rhys Sullivan committed 2 months ago

Commit d759daf1 parent 34663902 files · +523 −0Browse files

Unified · Split

2 files
  1. packages/core/sdk/src/tool-sync-schedule.test.ts +290 −0
  2. packages/core/sdk/src/tool-sync-schedule.ts +233 −0
packages/core/sdk/src/tool-sync-schedule.test.ts+290−0
@@ -0,0 +1,290 @@
1import { describe, expect, it } from "@effect/vitest";
2
3import {
4 TOOL_SYNC_BACKOFF_CEILING_MS,
5 TOOL_SYNC_CLAIM_LEASE_MS,
6 TOOL_SYNC_JITTER_MAX,
7 TOOL_SYNC_JITTER_MIN,
8 classifyToolSync,
9 decideToolSync,
10 isSyncEligible,
11 isToolSyncBackedOff,
12 isToolSyncClaimLive,
13 isToolSyncErrorKind,
14 isToolSyncParked,
15 scheduleAfterFailure,
16 scheduleAfterSuccess,
17 type ToolSyncCandidate,
18} from "./tool-sync-schedule";
19
20// The catalog sync lifecycle is a pure decision function, and these are the
21// decisions it exists to make: what a persisted catalog IS, whether anyone is
22// allowed to re-list it right now, and how long a failing connection is left
23// alone. Testing it here rather than only through the executor is what keeps
24// the ladder, the cap and the park rule honest without a database and a live
25// upstream in the loop.
26
27const NOW = 1_700_000_000_000;
28const TTL = 15 * 60 * 1000;
29
30/** A connection that has just synced successfully against a remote catalog. */
31const fresh = (overrides: Partial<ToolSyncCandidate> = {}): ToolSyncCandidate => ({
32 toolsSyncedAt: NOW - 1000,
33 toolsStaleAt: null,
34 toolsSyncClaimId: null,
35 toolsSyncClaimAt: null,
36 toolsSyncFailures: 0,
37 toolsSyncRetryAt: null,
38 toolsSyncErrorKind: null,
39 configRevisedAt: null,
40 remoteToolCatalog: true,
41 ttlMs: TTL,
42 ...overrides,
43});
44
45describe("isToolSyncErrorKind", () => {
46 it("accepts exactly the four kinds", () => {
47 expect(["auth", "unreachable", "protocol", "config"].every(isToolSyncErrorKind)).toBe(true);
48 });
49
50 it("rejects anything else a text column could hold", () => {
51 for (const value of ["", "AUTH", "authentication", null, undefined, 401, {}]) {
52 expect(isToolSyncErrorKind(value)).toBe(false);
53 }
54 });
55});
56
57describe("classifyToolSync", () => {
58 it("calls a never-synced, never-marked connection cold", () => {
59 expect(classifyToolSync(fresh({ toolsSyncedAt: null }), NOW)).toBe("cold");
60 });
61
62 it("calls a connection with a drift mark stale_marked, keeping its stamp", () => {
63 const candidate = fresh({ toolsSyncedAt: NOW - 60_000, toolsStaleAt: NOW - 1000 });
64 expect(classifyToolSync(candidate, NOW)).toBe("stale_marked");
65 });
66
67 it("still sees a drift mark that landed in the same millisecond as the stamp", () => {
68 // Date.now() has millisecond granularity, and a `tools/list_changed`
69 // arriving inside the millisecond it invalidates must not be swallowed.
70 expect(classifyToolSync(fresh({ toolsSyncedAt: NOW, toolsStaleAt: NOW }), NOW)).toBe(
71 "stale_marked",
72 );
73 });
74
75 it("calls a never-synced connection carrying a drift mark stale_marked, not cold", () => {
76 expect(classifyToolSync(fresh({ toolsSyncedAt: null, toolsStaleAt: NOW }), NOW)).toBe(
77 "stale_marked",
78 );
79 });
80
81 it("calls a catalog older than its integration's config revision config_revised", () => {
82 expect(
83 classifyToolSync(fresh({ toolsSyncedAt: NOW - 60_000, configRevisedAt: NOW - 30_000 }), NOW),
84 ).toBe("config_revised");
85 });
86
87 it("leaves a catalog newer than the config revision alone", () => {
88 expect(
89 classifyToolSync(fresh({ toolsSyncedAt: NOW - 30_000, configRevisedAt: NOW - 60_000 }), NOW),
90 ).toBe("fresh");
91 });
92
93 it("expires a remote catalog past the TTL", () => {
94 expect(classifyToolSync(fresh({ toolsSyncedAt: NOW - TTL - 1 }), NOW)).toBe("expired");
95 });
96
97 it("never expires a catalog the plugin does not list remotely", () => {
98 // An OpenAPI catalog is derived from a stored spec: time alone cannot make
99 // it wrong, and re-listing it on a timer is pure waste.
100 expect(
101 classifyToolSync(fresh({ toolsSyncedAt: NOW - TTL - 1, remoteToolCatalog: false }), NOW),
102 ).toBe("fresh");
103 });
104
105 it("never expires anything when the TTL is disabled", () => {
106 expect(classifyToolSync(fresh({ toolsSyncedAt: 0, ttlMs: null }), NOW)).toBe("fresh");
107 });
108
109 it("ranks an explicit drift signal above a config revision and the clock", () => {
110 const candidate = fresh({
111 toolsSyncedAt: NOW - TTL - 1,
112 toolsStaleAt: NOW,
113 configRevisedAt: NOW - 1,
114 });
115 expect(classifyToolSync(candidate, NOW)).toBe("stale_marked");
116 });
117
118 it("ranks a config revision above the clock", () => {
119 const candidate = fresh({ toolsSyncedAt: NOW - TTL - 1, configRevisedAt: NOW - 1 });
120 expect(classifyToolSync(candidate, NOW)).toBe("config_revised");
121 });
122});
123
124describe("isToolSyncClaimLive", () => {
125 it("is false with no claim", () => {
126 expect(isToolSyncClaimLive(fresh(), NOW)).toBe(false);
127 });
128
129 it("is true inside the lease", () => {
130 const candidate = fresh({
131 toolsSyncClaimId: "sync_abc",
132 toolsSyncClaimAt: NOW - TOOL_SYNC_CLAIM_LEASE_MS + 1,
133 });
134 expect(isToolSyncClaimLive(candidate, NOW)).toBe(true);
135 });
136
137 it("is false once the lease has run out", () => {
138 const candidate = fresh({
139 toolsSyncClaimId: "sync_abc",
140 toolsSyncClaimAt: NOW - TOOL_SYNC_CLAIM_LEASE_MS,
141 });
142 expect(isToolSyncClaimLive(candidate, NOW)).toBe(false);
143 });
144
145 it("treats a claim with no timestamp as dead rather than eternal", () => {
146 // Otherwise a half-written row would lock a connection out of every future
147 // refresh, forever, with nothing to expire it.
148 const candidate = fresh({ toolsSyncClaimId: "sync_abc", toolsSyncClaimAt: null });
149 expect(isToolSyncClaimLive(candidate, NOW)).toBe(false);
150 });
151});
152
153describe("isToolSyncBackedOff", () => {
154 it("holds a failing connection off until its retry instant", () => {
155 const candidate = fresh({ toolsSyncFailures: 2, toolsSyncRetryAt: NOW + 1 });
156 expect(isToolSyncBackedOff(candidate, NOW)).toBe(true);
157 expect(isToolSyncBackedOff(candidate, NOW + 1)).toBe(false);
158 });
159
160 it("ignores the retry instant a SUCCESS wrote", () => {
161 // After a success `tools_sync_retry_at` carries the freshness horizon, not
162 // a gate: a drift signal arriving inside the window must heal the catalog
163 // immediately instead of waiting the window out.
164 const candidate = fresh({
165 toolsSyncFailures: 0,
166 toolsSyncRetryAt: NOW + TTL,
167 toolsStaleAt: NOW,
168 });
169 expect(isToolSyncBackedOff(candidate, NOW)).toBe(false);
170 expect(isSyncEligible(candidate, NOW)).toBe(true);
171 });
172});
173
174describe("isToolSyncParked", () => {
175 it("parks only on an auth verdict", () => {
176 expect(isToolSyncParked(fresh({ toolsSyncErrorKind: "auth" }))).toBe(true);
177 for (const kind of ["unreachable", "protocol", "config", null] as const) {
178 expect(isToolSyncParked(fresh({ toolsSyncErrorKind: kind }))).toBe(false);
179 }
180 });
181});
182
183describe("isSyncEligible", () => {
184 it("passes a due connection with a clear ledger, reporting what made it due", () => {
185 expect(decideToolSync(fresh({ toolsSyncedAt: null }), NOW)).toEqual({
186 kind: "refresh",
187 trigger: "cold",
188 });
189 expect(isSyncEligible(fresh({ toolsSyncedAt: null }), NOW)).toBe(true);
190 });
191
192 it("skips a fresh connection", () => {
193 expect(decideToolSync(fresh(), NOW)).toEqual({ kind: "skip", reason: "fresh" });
194 expect(isSyncEligible(fresh(), NOW)).toBe(false);
195 });
196
197 it("skips a parked connection however overdue it is", () => {
198 const candidate = fresh({
199 toolsSyncedAt: null,
200 toolsSyncFailures: 9,
201 toolsSyncErrorKind: "auth",
202 toolsSyncRetryAt: NOW - 1,
203 });
204 expect(decideToolSync(candidate, NOW)).toEqual({ kind: "skip", reason: "parked" });
205 expect(isSyncEligible(candidate, NOW)).toBe(false);
206 });
207
208 it("skips a connection inside its backoff window", () => {
209 const candidate = fresh({
210 toolsSyncedAt: null,
211 toolsSyncFailures: 1,
212 toolsSyncErrorKind: "unreachable",
213 toolsSyncRetryAt: NOW + 1,
214 });
215 expect(decideToolSync(candidate, NOW)).toEqual({ kind: "skip", reason: "backoff" });
216 });
217
218 it("skips a connection another attempt is already holding", () => {
219 const candidate = fresh({
220 toolsSyncedAt: null,
221 toolsSyncClaimId: "sync_abc",
222 toolsSyncClaimAt: NOW,
223 });
224 expect(decideToolSync(candidate, NOW)).toEqual({ kind: "skip", reason: "claimed" });
225 });
226
227 it("re-admits a connection whose claimant died", () => {
228 const candidate = fresh({
229 toolsSyncedAt: null,
230 toolsSyncClaimId: "sync_abc",
231 toolsSyncClaimAt: NOW - TOOL_SYNC_CLAIM_LEASE_MS,
232 });
233 expect(isSyncEligible(candidate, NOW)).toBe(true);
234 });
235});
236
237describe("scheduleAfterSuccess", () => {
238 it("puts the next attempt one freshness window out", () => {
239 expect(scheduleAfterSuccess(NOW, TTL)).toBe(NOW + TTL);
240 });
241
242 it("schedules nothing when time-based re-sync is disabled", () => {
243 expect(scheduleAfterSuccess(NOW, null)).toBe(null);
244 });
245});
246
247describe("scheduleAfterFailure", () => {
248 const delay = (failures: number, jitter = 1, ttlMs = TTL) =>
249 scheduleAfterFailure(NOW, ttlMs, failures, jitter) - NOW;
250
251 it("doubles the wait with every consecutive failure", () => {
252 expect(delay(1)).toBe(TTL);
253 expect(delay(2)).toBe(2 * TTL);
254 expect(delay(3)).toBe(4 * TTL);
255 expect(delay(4)).toBe(8 * TTL);
256 });
257
258 it("caps the ladder rather than growing without bound", () => {
259 // A server dead for a week is still probed a few times a day, which is what
260 // makes recovery automatic.
261 expect(delay(20)).toBe(TOOL_SYNC_BACKOFF_CEILING_MS);
262 expect(delay(1000)).toBe(TOOL_SYNC_BACKOFF_CEILING_MS);
263 });
264
265 it("reads a failure count below one as the first failure", () => {
266 expect(delay(0)).toBe(TTL);
267 expect(delay(-5)).toBe(TTL);
268 });
269
270 it("stays within the jitter bounds at every rung", () => {
271 for (let failures = 1; failures <= 12; failures += 1) {
272 const base = Math.min(TOOL_SYNC_BACKOFF_CEILING_MS, TTL * 2 ** (failures - 1));
273 expect(delay(failures, TOOL_SYNC_JITTER_MIN)).toBe(Math.round(base * TOOL_SYNC_JITTER_MIN));
274 expect(delay(failures, TOOL_SYNC_JITTER_MAX)).toBe(Math.round(base * TOOL_SYNC_JITTER_MAX));
275 for (const jitter of [TOOL_SYNC_JITTER_MIN, 0.93, 1.07, TOOL_SYNC_JITTER_MAX]) {
276 const value = delay(failures, jitter);
277 expect(value).toBeGreaterThanOrEqual(Math.round(base * TOOL_SYNC_JITTER_MIN));
278 expect(value).toBeLessThanOrEqual(Math.round(base * TOOL_SYNC_JITTER_MAX));
279 }
280 }
281 });
282
283 it("returns a whole number of milliseconds for the bigint column", () => {
284 expect(Number.isInteger(scheduleAfterFailure(NOW, TTL, 3, 0.8137))).toBe(true);
285 });
286
287 it("scales the ladder to whatever base it is given", () => {
288 expect(delay(5, 1, 1000)).toBe(16_000);
289 });
290});
packages/core/sdk/src/tool-sync-schedule.ts+233−0
@@ -0,0 +1,233 @@
1// ---------------------------------------------------------------------------
2// Tool-catalog sync lifecycle — the pure state model behind every decision the
3// executor makes about re-listing a connection's tools.
4//
5// No storage, no clock, no randomness: the connection's columns arrive as a
6// value, `now` and the backoff jitter arrive as arguments, and every function
7// here is total. That is what makes the ladder, the cap and the eligibility
8// rules testable directly rather than through a database and a live upstream,
9// and it is what will let PR 3's background scheduler reuse the same rules
10// without reimplementing them against a different clock.
11// ---------------------------------------------------------------------------
12
13/** Why a catalog listing could not be completed, as a CLOSED set persisted on
14 * `connection.tools_sync_error_kind`.
15 *
16 * Structural on purpose: the previous mechanism was a `"Tool sync failing: "`
17 * prefix on the human-readable health detail, which meant every consumer that
18 * wanted to know "is this connection's catalog broken, and how" had to match a
19 * display string. These four values are the whole vocabulary:
20 *
21 * - `auth` the credential is rejected or absent. Retrying the same grant
22 * cannot change the verdict, so it PARKS the connection (see
23 * {@link isToolSyncParked}) until a human re-authorizes.
24 * - `unreachable` the upstream did not answer (DNS, connect, timeout).
25 * - `protocol` the upstream answered but the listing could not be read.
26 * - `config` the integration's own configuration cannot produce a listing
27 * (unparseable spec, disabled transport, missing endpoint). */
28export type ToolSyncErrorKind = "auth" | "unreachable" | "protocol" | "config";
29
30export const TOOL_SYNC_ERROR_KINDS = [
31 "auth",
32 "unreachable",
33 "protocol",
34 "config",
35] as const satisfies readonly ToolSyncErrorKind[];
36
37/** Narrow a persisted `tools_sync_error_kind` column back to the closed set.
38 * The column is plain text, so a value written by a newer build (or edited by
39 * hand) has to be parsed rather than trusted. */
40export const isToolSyncErrorKind = (value: unknown): value is ToolSyncErrorKind =>
41 typeof value === "string" && (TOOL_SYNC_ERROR_KINDS as readonly string[]).includes(value);
42
43/**
44 * What a connection's persisted catalog is, relative to now.
45 *
46 * Every non-`fresh` value is exactly the reason a read has to re-list, so this
47 * doubles as the refresh trigger reported on the `executor.tools.sync` span.
48 *
49 * `cold` and `stale_marked` were one value before the `tools_stale_at` column
50 * existed: `connections.markToolsStale` cleared `tools_synced_at`, which made a
51 * connection that had never synced indistinguishable from one whose catalog was
52 * invalidated mid-invocation, and destroyed the last-verified timestamp in the
53 * process.
54 */
55export type ToolSyncState = "fresh" | "cold" | "stale_marked" | "config_revised" | "expired";
56
57/** Why an otherwise-due connection is not being refreshed right now. Enumerable
58 * so the read's span can count each reason. */
59export type ToolSyncSkip = "fresh" | "parked" | "backoff" | "claimed";
60
61/**
62 * A connection's sync lifecycle columns, plus the two facts that live on its
63 * integration and plugin (`config_revised_at`, whether the plugin lists a live
64 * remote catalog) and the executor's freshness window.
65 *
66 * Decoded from the row by the caller: bigint columns arrive as `bigint | null`
67 * from the driver, and the text `tools_sync_error_kind` has to pass
68 * {@link isToolSyncErrorKind} before it gets here.
69 */
70export interface ToolSyncCandidate {
71 /** Epoch ms of the last AUTHORITATIVE listing. Never stamped by a failed or
72 * incomplete one — a month-dead server must not read as "synced 30s ago". */
73 readonly toolsSyncedAt: number | null;
74 /** Epoch ms an event declared the persisted catalog drifted. */
75 readonly toolsStaleAt: number | null;
76 /** The nonce of the refresh attempt currently holding the write lease. */
77 readonly toolsSyncClaimId: string | null;
78 /** Epoch ms the claim was taken, against which the lease expires. */
79 readonly toolsSyncClaimAt: number | null;
80 /** CONSECUTIVE failed listings; zeroed by an authoritative one. */
81 readonly toolsSyncFailures: number;
82 /** Epoch ms of the earliest next attempt. */
83 readonly toolsSyncRetryAt: number | null;
84 readonly toolsSyncErrorKind: ToolSyncErrorKind | null;
85 /** The integration's `config_revised_at`: its last tool-affecting config
86 * change. */
87 readonly configRevisedAt: number | null;
88 /** The plugin lists a live remote catalog (an MCP server, whose tool set can
89 * change with no executor-visible signal), so time alone can expire it. */
90 readonly remoteToolCatalog: boolean;
91 /** The freshness window; `null` disables time-based expiry entirely. */
92 readonly ttlMs: number | null;
93}
94
95/**
96 * How long a refresh claim stays valid.
97 *
98 * A lease rather than a lock: the claimant can be a Workers isolate that is
99 * evicted mid-handshake, and nothing would ever release its claim. Long enough
100 * to cover a slow upstream listing (discovery is bounded well under this),
101 * short enough that a lost claimant costs one wasted freshness window.
102 */
103export const TOOL_SYNC_CLAIM_LEASE_MS = 60_000;
104
105/** The longest a failing connection is left alone. Past this the ladder stops
106 * doubling: a server that has been dead for a week still gets probed a few
107 * times a day, which is what makes recovery automatic. */
108export const TOOL_SYNC_BACKOFF_CEILING_MS = 6 * 60 * 60 * 1000;
109
110/** Bounds of the multiplicative jitter applied to every backoff delay, so a
111 * fleet that lost the same upstream at the same moment does not re-dial it in
112 * lockstep. The factor itself is supplied by the caller — this module owns no
113 * randomness. */
114export const TOOL_SYNC_JITTER_MIN = 0.8;
115export const TOOL_SYNC_JITTER_MAX = 1.2;
116
117/**
118 * Classify a connection's catalog. Total, and ordered by authority: an explicit
119 * drift signal outranks a config revision, which outranks the clock.
120 *
121 * The stale comparison is `>=`, not `>`: `Date.now()` has millisecond
122 * granularity, and a `tools/list_changed` notification arriving in the same
123 * millisecond as the stamp it invalidates must not be swallowed.
124 */
125export const classifyToolSync = (candidate: ToolSyncCandidate, now: number): ToolSyncState => {
126 const syncedAt = candidate.toolsSyncedAt;
127 // A never-synced connection sorts before every stamp, which is why the
128 // comparisons below read through `?? 0` rather than special-casing null.
129 if (candidate.toolsStaleAt !== null && candidate.toolsStaleAt >= (syncedAt ?? 0)) {
130 return "stale_marked";
131 }
132 if (syncedAt === null) return "cold";
133 if (candidate.configRevisedAt !== null && syncedAt < candidate.configRevisedAt) {
134 return "config_revised";
135 }
136 if (candidate.remoteToolCatalog && candidate.ttlMs !== null && syncedAt < now - candidate.ttlMs) {
137 return "expired";
138 }
139 return "fresh";
140};
141
142/** Whether another attempt currently holds the write lease. A claim with no
143 * `claim_at` is treated as dead rather than eternal, so a half-written row can
144 * never wedge a connection out of every future refresh. */
145export const isToolSyncClaimLive = (candidate: ToolSyncCandidate, now: number): boolean =>
146 candidate.toolsSyncClaimId !== null &&
147 candidate.toolsSyncClaimAt !== null &&
148 now - candidate.toolsSyncClaimAt < TOOL_SYNC_CLAIM_LEASE_MS;
149
150/**
151 * Whether the failure ladder is still holding this connection off.
152 *
153 * Gated on `failures > 0` deliberately. `tools_sync_retry_at` is also written
154 * after a SUCCESS, where it records the TTL horizon so a scheduler can order
155 * work by one column; treating that as a gate would make a `list_changed`
156 * notification wait out the freshness window before it could heal the catalog.
157 * An event-driven drift re-lists immediately; only a failing one waits.
158 */
159export const isToolSyncBackedOff = (candidate: ToolSyncCandidate, now: number): boolean =>
160 candidate.toolsSyncFailures > 0 &&
161 candidate.toolsSyncRetryAt !== null &&
162 now < candidate.toolsSyncRetryAt;
163
164/** Whether the connection is parked: its credential is rejected, and no amount
165 * of retrying the same grant will change that. Cleared by the human-initiated
166 * paths that can actually fix it (explicit refresh, connection update, an
167 * OAuth re-mint, a healthy probe, a fresh drift signal). */
168export const isToolSyncParked = (candidate: ToolSyncCandidate): boolean =>
169 candidate.toolsSyncErrorKind === "auth";
170
171/**
172 * Refresh this connection, or pass it over and say why. ONE decision rather
173 * than a predicate plus a re-classification, so a caller that needs both the
174 * verdict and the trigger cannot end up asking two questions with two answers.
175 *
176 * Every clause that can stop a refresh lives here, and that is the point: a
177 * read's inline refresh, an explicit signal and a background sweep all consult
178 * this, so "when do we dial an upstream" has one answer rather than one per
179 * caller. A `connection.status` lifecycle (disabled, revoked) belongs here too
180 * when it lands, as one more skip clause — putting it at a call site is what
181 * would reintroduce the divergence this exists to prevent.
182 *
183 * Ordered cheapest-and-most-final first, so the reason an operator sees on the
184 * span is the one they would name. A live claim observed on the row is only a
185 * pre-filter: the binding decision is the compare-and-set the caller runs
186 * against the database, and two readers that both see a free claim here still
187 * resolve to one refresh.
188 */
189export type ToolSyncDecision =
190 | { readonly kind: "skip"; readonly reason: ToolSyncSkip }
191 | { readonly kind: "refresh"; readonly trigger: Exclude<ToolSyncState, "fresh"> };
192
193export const decideToolSync = (candidate: ToolSyncCandidate, now: number): ToolSyncDecision => {
194 const state = classifyToolSync(candidate, now);
195 if (state === "fresh") return { kind: "skip", reason: "fresh" };
196 if (isToolSyncParked(candidate)) return { kind: "skip", reason: "parked" };
197 if (isToolSyncBackedOff(candidate, now)) return { kind: "skip", reason: "backoff" };
198 if (isToolSyncClaimLive(candidate, now)) return { kind: "skip", reason: "claimed" };
199 return { kind: "refresh", trigger: state };
200};
201
202/** THE eligibility predicate, for callers that only need the yes/no. */
203export const isSyncEligible = (candidate: ToolSyncCandidate, now: number): boolean =>
204 decideToolSync(candidate, now).kind === "refresh";
205
206/** The earliest next attempt after an AUTHORITATIVE listing: one freshness
207 * window out, or `null` when time-based re-sync is disabled. Not a gate (see
208 * {@link isToolSyncBackedOff}) — it is the horizon a scheduler sorts by. */
209export const scheduleAfterSuccess = (now: number, ttlMs: number | null): number | null =>
210 ttlMs === null ? null : now + ttlMs;
211
212/**
213 * The earliest next attempt after a FAILED listing: `ttlMs × 2^(failures−1)`,
214 * capped at {@link TOOL_SYNC_BACKOFF_CEILING_MS}, scaled by `jitter`.
215 *
216 * `failures` is the count INCLUDING the failure being recorded, so the first
217 * failure waits one plain TTL; values below 1 are read as 1 rather than halving
218 * the first delay. `jitter` belongs to the caller and is expected in
219 * [{@link TOOL_SYNC_JITTER_MIN}, {@link TOOL_SYNC_JITTER_MAX}]; it is not
220 * clamped here, because a call site that passes something else has a bug worth
221 * seeing rather than silently correcting.
222 *
223 * Rounded: the result lands in a bigint column.
224 */
225export const scheduleAfterFailure = (
226 now: number,
227 ttlMs: number,
228 failures: number,
229 jitter: number,
230): number =>
231 Math.round(
232 now + Math.min(TOOL_SYNC_BACKOFF_CEILING_MS, ttlMs * 2 ** (Math.max(1, failures) - 1)) * jitter,
233 );