Add the pure tool-catalog sync lifecycle model

Rhys Sullivan committed 2 months ago

Commit d759daf1 parent 34663902 files · +523 −0Browse files

Unified · Split

2 files

packages/core/sdk/src/tool-sync-schedule.test.ts +290 −0
packages/core/sdk/src/tool-sync-schedule.ts +233 −0

packages/core/sdk/src/tool-sync-schedule.test.ts+290−0

@@ -0,0 +1,290 @@
	1	+import { describe, expect, it } from "@effect/vitest";
	2	+
	3	+import {
	4	+ TOOL_SYNC_BACKOFF_CEILING_MS,
	5	+ TOOL_SYNC_CLAIM_LEASE_MS,
	6	+ TOOL_SYNC_JITTER_MAX,
	7	+ TOOL_SYNC_JITTER_MIN,
	8	+ classifyToolSync,
	9	+ decideToolSync,
	10	+ isSyncEligible,
	11	+ isToolSyncBackedOff,
	12	+ isToolSyncClaimLive,
	13	+ isToolSyncErrorKind,
	14	+ isToolSyncParked,
	15	+ scheduleAfterFailure,
	16	+ scheduleAfterSuccess,
	17	+ type ToolSyncCandidate,
	18	+} from "./tool-sync-schedule";
	19	+
	20	+// The catalog sync lifecycle is a pure decision function, and these are the
	21	+// decisions it exists to make: what a persisted catalog IS, whether anyone is
	22	+// allowed to re-list it right now, and how long a failing connection is left
	23	+// alone. Testing it here rather than only through the executor is what keeps
	24	+// the ladder, the cap and the park rule honest without a database and a live
	25	+// upstream in the loop.
	26	+
	27	+const NOW = 1_700_000_000_000;
	28	+const TTL = 15 * 60 * 1000;
	29	+
	30	+/** A connection that has just synced successfully against a remote catalog. */
	31	+const fresh = (overrides: Partial<ToolSyncCandidate> = {}): ToolSyncCandidate => ({
	32	+ toolsSyncedAt: NOW - 1000,
	33	+ toolsStaleAt: null,
	34	+ toolsSyncClaimId: null,
	35	+ toolsSyncClaimAt: null,
	36	+ toolsSyncFailures: 0,
	37	+ toolsSyncRetryAt: null,
	38	+ toolsSyncErrorKind: null,
	39	+ configRevisedAt: null,
	40	+ remoteToolCatalog: true,
	41	+ ttlMs: TTL,
	42	+ ...overrides,
	43	+});
	44	+
	45	+describe("isToolSyncErrorKind", () => {
	46	+ it("accepts exactly the four kinds", () => {
	47	+ expect(["auth", "unreachable", "protocol", "config"].every(isToolSyncErrorKind)).toBe(true);
	48	+ });
	49	+
	50	+ it("rejects anything else a text column could hold", () => {
	51	+ for (const value of ["", "AUTH", "authentication", null, undefined, 401, {}]) {
	52	+ expect(isToolSyncErrorKind(value)).toBe(false);
	53	+ }
	54	+ });
	55	+});
	56	+
	57	+describe("classifyToolSync", () => {
	58	+ it("calls a never-synced, never-marked connection cold", () => {
	59	+ expect(classifyToolSync(fresh({ toolsSyncedAt: null }), NOW)).toBe("cold");
	60	+ });
	61	+
	62	+ it("calls a connection with a drift mark stale_marked, keeping its stamp", () => {
	63	+ const candidate = fresh({ toolsSyncedAt: NOW - 60_000, toolsStaleAt: NOW - 1000 });
	64	+ expect(classifyToolSync(candidate, NOW)).toBe("stale_marked");
	65	+ });
	66	+
	67	+ it("still sees a drift mark that landed in the same millisecond as the stamp", () => {
	68	+ // Date.now() has millisecond granularity, and a `tools/list_changed`
	69	+ // arriving inside the millisecond it invalidates must not be swallowed.
	70	+ expect(classifyToolSync(fresh({ toolsSyncedAt: NOW, toolsStaleAt: NOW }), NOW)).toBe(
	71	+ "stale_marked",
	72	+ );
	73	+ });
	74	+
	75	+ it("calls a never-synced connection carrying a drift mark stale_marked, not cold", () => {
	76	+ expect(classifyToolSync(fresh({ toolsSyncedAt: null, toolsStaleAt: NOW }), NOW)).toBe(
	77	+ "stale_marked",
	78	+ );
	79	+ });
	80	+
	81	+ it("calls a catalog older than its integration's config revision config_revised", () => {
	82	+ expect(
	83	+ classifyToolSync(fresh({ toolsSyncedAt: NOW - 60_000, configRevisedAt: NOW - 30_000 }), NOW),
	84	+ ).toBe("config_revised");
	85	+ });
	86	+
	87	+ it("leaves a catalog newer than the config revision alone", () => {
	88	+ expect(
	89	+ classifyToolSync(fresh({ toolsSyncedAt: NOW - 30_000, configRevisedAt: NOW - 60_000 }), NOW),
	90	+ ).toBe("fresh");
	91	+ });
	92	+
	93	+ it("expires a remote catalog past the TTL", () => {
	94	+ expect(classifyToolSync(fresh({ toolsSyncedAt: NOW - TTL - 1 }), NOW)).toBe("expired");
	95	+ });
	96	+
	97	+ it("never expires a catalog the plugin does not list remotely", () => {
	98	+ // An OpenAPI catalog is derived from a stored spec: time alone cannot make
	99	+ // it wrong, and re-listing it on a timer is pure waste.
	100	+ expect(
	101	+ classifyToolSync(fresh({ toolsSyncedAt: NOW - TTL - 1, remoteToolCatalog: false }), NOW),
	102	+ ).toBe("fresh");
	103	+ });
	104	+
	105	+ it("never expires anything when the TTL is disabled", () => {
	106	+ expect(classifyToolSync(fresh({ toolsSyncedAt: 0, ttlMs: null }), NOW)).toBe("fresh");
	107	+ });
	108	+
	109	+ it("ranks an explicit drift signal above a config revision and the clock", () => {
	110	+ const candidate = fresh({
	111	+ toolsSyncedAt: NOW - TTL - 1,
	112	+ toolsStaleAt: NOW,
	113	+ configRevisedAt: NOW - 1,
	114	+ });
	115	+ expect(classifyToolSync(candidate, NOW)).toBe("stale_marked");
	116	+ });
	117	+
	118	+ it("ranks a config revision above the clock", () => {
	119	+ const candidate = fresh({ toolsSyncedAt: NOW - TTL - 1, configRevisedAt: NOW - 1 });
	120	+ expect(classifyToolSync(candidate, NOW)).toBe("config_revised");
	121	+ });
	122	+});
	123	+
	124	+describe("isToolSyncClaimLive", () => {
	125	+ it("is false with no claim", () => {
	126	+ expect(isToolSyncClaimLive(fresh(), NOW)).toBe(false);
	127	+ });
	128	+
	129	+ it("is true inside the lease", () => {
	130	+ const candidate = fresh({
	131	+ toolsSyncClaimId: "sync_abc",
	132	+ toolsSyncClaimAt: NOW - TOOL_SYNC_CLAIM_LEASE_MS + 1,
	133	+ });
	134	+ expect(isToolSyncClaimLive(candidate, NOW)).toBe(true);
	135	+ });
	136	+
	137	+ it("is false once the lease has run out", () => {
	138	+ const candidate = fresh({
	139	+ toolsSyncClaimId: "sync_abc",
	140	+ toolsSyncClaimAt: NOW - TOOL_SYNC_CLAIM_LEASE_MS,
	141	+ });
	142	+ expect(isToolSyncClaimLive(candidate, NOW)).toBe(false);
	143	+ });
	144	+
	145	+ it("treats a claim with no timestamp as dead rather than eternal", () => {
	146	+ // Otherwise a half-written row would lock a connection out of every future
	147	+ // refresh, forever, with nothing to expire it.
	148	+ const candidate = fresh({ toolsSyncClaimId: "sync_abc", toolsSyncClaimAt: null });
	149	+ expect(isToolSyncClaimLive(candidate, NOW)).toBe(false);
	150	+ });
	151	+});
	152	+
	153	+describe("isToolSyncBackedOff", () => {
	154	+ it("holds a failing connection off until its retry instant", () => {
	155	+ const candidate = fresh({ toolsSyncFailures: 2, toolsSyncRetryAt: NOW + 1 });
	156	+ expect(isToolSyncBackedOff(candidate, NOW)).toBe(true);
	157	+ expect(isToolSyncBackedOff(candidate, NOW + 1)).toBe(false);
	158	+ });
	159	+
	160	+ it("ignores the retry instant a SUCCESS wrote", () => {
	161	+ // After a success `tools_sync_retry_at` carries the freshness horizon, not
	162	+ // a gate: a drift signal arriving inside the window must heal the catalog
	163	+ // immediately instead of waiting the window out.
	164	+ const candidate = fresh({
	165	+ toolsSyncFailures: 0,
	166	+ toolsSyncRetryAt: NOW + TTL,
	167	+ toolsStaleAt: NOW,
	168	+ });
	169	+ expect(isToolSyncBackedOff(candidate, NOW)).toBe(false);
	170	+ expect(isSyncEligible(candidate, NOW)).toBe(true);
	171	+ });
	172	+});
	173	+
	174	+describe("isToolSyncParked", () => {
	175	+ it("parks only on an auth verdict", () => {
	176	+ expect(isToolSyncParked(fresh({ toolsSyncErrorKind: "auth" }))).toBe(true);
	177	+ for (const kind of ["unreachable", "protocol", "config", null] as const) {
	178	+ expect(isToolSyncParked(fresh({ toolsSyncErrorKind: kind }))).toBe(false);
	179	+ }
	180	+ });
	181	+});
	182	+
	183	+describe("isSyncEligible", () => {
	184	+ it("passes a due connection with a clear ledger, reporting what made it due", () => {
	185	+ expect(decideToolSync(fresh({ toolsSyncedAt: null }), NOW)).toEqual({
	186	+ kind: "refresh",
	187	+ trigger: "cold",
	188	+ });
	189	+ expect(isSyncEligible(fresh({ toolsSyncedAt: null }), NOW)).toBe(true);
	190	+ });
	191	+
	192	+ it("skips a fresh connection", () => {
	193	+ expect(decideToolSync(fresh(), NOW)).toEqual({ kind: "skip", reason: "fresh" });
	194	+ expect(isSyncEligible(fresh(), NOW)).toBe(false);
	195	+ });
	196	+
	197	+ it("skips a parked connection however overdue it is", () => {
	198	+ const candidate = fresh({
	199	+ toolsSyncedAt: null,
	200	+ toolsSyncFailures: 9,
	201	+ toolsSyncErrorKind: "auth",
	202	+ toolsSyncRetryAt: NOW - 1,
	203	+ });
	204	+ expect(decideToolSync(candidate, NOW)).toEqual({ kind: "skip", reason: "parked" });
	205	+ expect(isSyncEligible(candidate, NOW)).toBe(false);
	206	+ });
	207	+
	208	+ it("skips a connection inside its backoff window", () => {
	209	+ const candidate = fresh({
	210	+ toolsSyncedAt: null,
	211	+ toolsSyncFailures: 1,
	212	+ toolsSyncErrorKind: "unreachable",
	213	+ toolsSyncRetryAt: NOW + 1,
	214	+ });
	215	+ expect(decideToolSync(candidate, NOW)).toEqual({ kind: "skip", reason: "backoff" });
	216	+ });
	217	+
	218	+ it("skips a connection another attempt is already holding", () => {
	219	+ const candidate = fresh({
	220	+ toolsSyncedAt: null,
	221	+ toolsSyncClaimId: "sync_abc",
	222	+ toolsSyncClaimAt: NOW,
	223	+ });
	224	+ expect(decideToolSync(candidate, NOW)).toEqual({ kind: "skip", reason: "claimed" });
	225	+ });
	226	+
	227	+ it("re-admits a connection whose claimant died", () => {
	228	+ const candidate = fresh({
	229	+ toolsSyncedAt: null,
	230	+ toolsSyncClaimId: "sync_abc",
	231	+ toolsSyncClaimAt: NOW - TOOL_SYNC_CLAIM_LEASE_MS,
	232	+ });
	233	+ expect(isSyncEligible(candidate, NOW)).toBe(true);
	234	+ });
	235	+});
	236	+
	237	+describe("scheduleAfterSuccess", () => {
	238	+ it("puts the next attempt one freshness window out", () => {
	239	+ expect(scheduleAfterSuccess(NOW, TTL)).toBe(NOW + TTL);
	240	+ });
	241	+
	242	+ it("schedules nothing when time-based re-sync is disabled", () => {
	243	+ expect(scheduleAfterSuccess(NOW, null)).toBe(null);
	244	+ });
	245	+});
	246	+
	247	+describe("scheduleAfterFailure", () => {
	248	+ const delay = (failures: number, jitter = 1, ttlMs = TTL) =>
	249	+ scheduleAfterFailure(NOW, ttlMs, failures, jitter) - NOW;
	250	+
	251	+ it("doubles the wait with every consecutive failure", () => {
	252	+ expect(delay(1)).toBe(TTL);
	253	+ expect(delay(2)).toBe(2 * TTL);
	254	+ expect(delay(3)).toBe(4 * TTL);
	255	+ expect(delay(4)).toBe(8 * TTL);
	256	+ });
	257	+
	258	+ it("caps the ladder rather than growing without bound", () => {
	259	+ // A server dead for a week is still probed a few times a day, which is what
	260	+ // makes recovery automatic.
	261	+ expect(delay(20)).toBe(TOOL_SYNC_BACKOFF_CEILING_MS);
	262	+ expect(delay(1000)).toBe(TOOL_SYNC_BACKOFF_CEILING_MS);
	263	+ });
	264	+
	265	+ it("reads a failure count below one as the first failure", () => {
	266	+ expect(delay(0)).toBe(TTL);
	267	+ expect(delay(-5)).toBe(TTL);
	268	+ });
	269	+
	270	+ it("stays within the jitter bounds at every rung", () => {
	271	+ for (let failures = 1; failures <= 12; failures += 1) {
	272	+ const base = Math.min(TOOL_SYNC_BACKOFF_CEILING_MS, TTL * 2 ** (failures - 1));
	273	+ expect(delay(failures, TOOL_SYNC_JITTER_MIN)).toBe(Math.round(base * TOOL_SYNC_JITTER_MIN));
	274	+ expect(delay(failures, TOOL_SYNC_JITTER_MAX)).toBe(Math.round(base * TOOL_SYNC_JITTER_MAX));
	275	+ for (const jitter of [TOOL_SYNC_JITTER_MIN, 0.93, 1.07, TOOL_SYNC_JITTER_MAX]) {
	276	+ const value = delay(failures, jitter);
	277	+ expect(value).toBeGreaterThanOrEqual(Math.round(base * TOOL_SYNC_JITTER_MIN));
	278	+ expect(value).toBeLessThanOrEqual(Math.round(base * TOOL_SYNC_JITTER_MAX));
	279	+ }
	280	+ }
	281	+ });
	282	+
	283	+ it("returns a whole number of milliseconds for the bigint column", () => {
	284	+ expect(Number.isInteger(scheduleAfterFailure(NOW, TTL, 3, 0.8137))).toBe(true);
	285	+ });
	286	+
	287	+ it("scales the ladder to whatever base it is given", () => {
	288	+ expect(delay(5, 1, 1000)).toBe(16_000);
	289	+ });
	290	+});

packages/core/sdk/src/tool-sync-schedule.ts+233−0

@@ -0,0 +1,233 @@
	1	+// ---------------------------------------------------------------------------
	2	+// Tool-catalog sync lifecycle — the pure state model behind every decision the
	3	+// executor makes about re-listing a connection's tools.
	4	+//
	5	+// No storage, no clock, no randomness: the connection's columns arrive as a
	6	+// value, `now` and the backoff jitter arrive as arguments, and every function
	7	+// here is total. That is what makes the ladder, the cap and the eligibility
	8	+// rules testable directly rather than through a database and a live upstream,
	9	+// and it is what will let PR 3's background scheduler reuse the same rules
	10	+// without reimplementing them against a different clock.
	11	+// ---------------------------------------------------------------------------
	12	+
	13	+/** Why a catalog listing could not be completed, as a CLOSED set persisted on
	14	+ * `connection.tools_sync_error_kind`.
	15	+ *
	16	+ * Structural on purpose: the previous mechanism was a `"Tool sync failing: "`
	17	+ * prefix on the human-readable health detail, which meant every consumer that
	18	+ * wanted to know "is this connection's catalog broken, and how" had to match a
	19	+ * display string. These four values are the whole vocabulary:
	20	+ *
	21	+ * - `auth` the credential is rejected or absent. Retrying the same grant
	22	+ * cannot change the verdict, so it PARKS the connection (see
	23	+ * {@link isToolSyncParked}) until a human re-authorizes.
	24	+ * - `unreachable` the upstream did not answer (DNS, connect, timeout).
	25	+ * - `protocol` the upstream answered but the listing could not be read.
	26	+ * - `config` the integration's own configuration cannot produce a listing
	27	+ * (unparseable spec, disabled transport, missing endpoint). */
	28	+export type ToolSyncErrorKind = "auth" \| "unreachable" \| "protocol" \| "config";
	29	+
	30	+export const TOOL_SYNC_ERROR_KINDS = [
	31	+ "auth",
	32	+ "unreachable",
	33	+ "protocol",
	34	+ "config",
	35	+] as const satisfies readonly ToolSyncErrorKind[];
	36	+
	37	+/** Narrow a persisted `tools_sync_error_kind` column back to the closed set.
	38	+ * The column is plain text, so a value written by a newer build (or edited by
	39	+ * hand) has to be parsed rather than trusted. */
	40	+export const isToolSyncErrorKind = (value: unknown): value is ToolSyncErrorKind =>
	41	+ typeof value === "string" && (TOOL_SYNC_ERROR_KINDS as readonly string[]).includes(value);
	42	+
	43	+/**
	44	+ * What a connection's persisted catalog is, relative to now.
	45	+ *
	46	+ * Every non-`fresh` value is exactly the reason a read has to re-list, so this
	47	+ * doubles as the refresh trigger reported on the `executor.tools.sync` span.
	48	+ *
	49	+ * `cold` and `stale_marked` were one value before the `tools_stale_at` column
	50	+ * existed: `connections.markToolsStale` cleared `tools_synced_at`, which made a
	51	+ * connection that had never synced indistinguishable from one whose catalog was
	52	+ * invalidated mid-invocation, and destroyed the last-verified timestamp in the
	53	+ * process.
	54	+ */
	55	+export type ToolSyncState = "fresh" \| "cold" \| "stale_marked" \| "config_revised" \| "expired";
	56	+
	57	+/** Why an otherwise-due connection is not being refreshed right now. Enumerable
	58	+ * so the read's span can count each reason. */
	59	+export type ToolSyncSkip = "fresh" \| "parked" \| "backoff" \| "claimed";
	60	+
	61	+/**
	62	+ * A connection's sync lifecycle columns, plus the two facts that live on its
	63	+ * integration and plugin (`config_revised_at`, whether the plugin lists a live
	64	+ * remote catalog) and the executor's freshness window.
	65	+ *
	66	+ * Decoded from the row by the caller: bigint columns arrive as `bigint \| null`
	67	+ * from the driver, and the text `tools_sync_error_kind` has to pass
	68	+ * {@link isToolSyncErrorKind} before it gets here.
	69	+ */
	70	+export interface ToolSyncCandidate {
	71	+ /** Epoch ms of the last AUTHORITATIVE listing. Never stamped by a failed or
	72	+ * incomplete one — a month-dead server must not read as "synced 30s ago". */
	73	+ readonly toolsSyncedAt: number \| null;
	74	+ /** Epoch ms an event declared the persisted catalog drifted. */
	75	+ readonly toolsStaleAt: number \| null;
	76	+ /** The nonce of the refresh attempt currently holding the write lease. */
	77	+ readonly toolsSyncClaimId: string \| null;
	78	+ /** Epoch ms the claim was taken, against which the lease expires. */
	79	+ readonly toolsSyncClaimAt: number \| null;
	80	+ /** CONSECUTIVE failed listings; zeroed by an authoritative one. */
	81	+ readonly toolsSyncFailures: number;
	82	+ /** Epoch ms of the earliest next attempt. */
	83	+ readonly toolsSyncRetryAt: number \| null;
	84	+ readonly toolsSyncErrorKind: ToolSyncErrorKind \| null;
	85	+ /** The integration's `config_revised_at`: its last tool-affecting config
	86	+ * change. */
	87	+ readonly configRevisedAt: number \| null;
	88	+ /** The plugin lists a live remote catalog (an MCP server, whose tool set can
	89	+ * change with no executor-visible signal), so time alone can expire it. */
	90	+ readonly remoteToolCatalog: boolean;
	91	+ /** The freshness window; `null` disables time-based expiry entirely. */
	92	+ readonly ttlMs: number \| null;
	93	+}
	94	+
	95	+/**
	96	+ * How long a refresh claim stays valid.
	97	+ *
	98	+ * A lease rather than a lock: the claimant can be a Workers isolate that is
	99	+ * evicted mid-handshake, and nothing would ever release its claim. Long enough
	100	+ * to cover a slow upstream listing (discovery is bounded well under this),
	101	+ * short enough that a lost claimant costs one wasted freshness window.
	102	+ */
	103	+export const TOOL_SYNC_CLAIM_LEASE_MS = 60_000;
	104	+
	105	+/** The longest a failing connection is left alone. Past this the ladder stops
	106	+ * doubling: a server that has been dead for a week still gets probed a few
	107	+ * times a day, which is what makes recovery automatic. */
	108	+export const TOOL_SYNC_BACKOFF_CEILING_MS = 6 * 60 * 60 * 1000;
	109	+
	110	+/** Bounds of the multiplicative jitter applied to every backoff delay, so a
	111	+ * fleet that lost the same upstream at the same moment does not re-dial it in
	112	+ * lockstep. The factor itself is supplied by the caller — this module owns no
	113	+ * randomness. */
	114	+export const TOOL_SYNC_JITTER_MIN = 0.8;
	115	+export const TOOL_SYNC_JITTER_MAX = 1.2;
	116	+
	117	+/**
	118	+ * Classify a connection's catalog. Total, and ordered by authority: an explicit
	119	+ * drift signal outranks a config revision, which outranks the clock.
	120	+ *
	121	+ * The stale comparison is `>=`, not `>`: `Date.now()` has millisecond
	122	+ * granularity, and a `tools/list_changed` notification arriving in the same
	123	+ * millisecond as the stamp it invalidates must not be swallowed.
	124	+ */
	125	+export const classifyToolSync = (candidate: ToolSyncCandidate, now: number): ToolSyncState => {
	126	+ const syncedAt = candidate.toolsSyncedAt;
	127	+ // A never-synced connection sorts before every stamp, which is why the
	128	+ // comparisons below read through `?? 0` rather than special-casing null.
	129	+ if (candidate.toolsStaleAt !== null && candidate.toolsStaleAt >= (syncedAt ?? 0)) {
	130	+ return "stale_marked";
	131	+ }
	132	+ if (syncedAt === null) return "cold";
	133	+ if (candidate.configRevisedAt !== null && syncedAt < candidate.configRevisedAt) {
	134	+ return "config_revised";
	135	+ }
	136	+ if (candidate.remoteToolCatalog && candidate.ttlMs !== null && syncedAt < now - candidate.ttlMs) {
	137	+ return "expired";
	138	+ }
	139	+ return "fresh";
	140	+};
	141	+
	142	+/** Whether another attempt currently holds the write lease. A claim with no
	143	+ * `claim_at` is treated as dead rather than eternal, so a half-written row can
	144	+ * never wedge a connection out of every future refresh. */
	145	+export const isToolSyncClaimLive = (candidate: ToolSyncCandidate, now: number): boolean =>
	146	+ candidate.toolsSyncClaimId !== null &&
	147	+ candidate.toolsSyncClaimAt !== null &&
	148	+ now - candidate.toolsSyncClaimAt < TOOL_SYNC_CLAIM_LEASE_MS;
	149	+
	150	+/**
	151	+ * Whether the failure ladder is still holding this connection off.
	152	+ *
	153	+ * Gated on `failures > 0` deliberately. `tools_sync_retry_at` is also written
	154	+ * after a SUCCESS, where it records the TTL horizon so a scheduler can order
	155	+ * work by one column; treating that as a gate would make a `list_changed`
	156	+ * notification wait out the freshness window before it could heal the catalog.
	157	+ * An event-driven drift re-lists immediately; only a failing one waits.
	158	+ */
	159	+export const isToolSyncBackedOff = (candidate: ToolSyncCandidate, now: number): boolean =>
	160	+ candidate.toolsSyncFailures > 0 &&
	161	+ candidate.toolsSyncRetryAt !== null &&
	162	+ now < candidate.toolsSyncRetryAt;
	163	+
	164	+/** Whether the connection is parked: its credential is rejected, and no amount
	165	+ * of retrying the same grant will change that. Cleared by the human-initiated
	166	+ * paths that can actually fix it (explicit refresh, connection update, an
	167	+ * OAuth re-mint, a healthy probe, a fresh drift signal). */
	168	+export const isToolSyncParked = (candidate: ToolSyncCandidate): boolean =>
	169	+ candidate.toolsSyncErrorKind === "auth";
	170	+
	171	+/**
	172	+ * Refresh this connection, or pass it over and say why. ONE decision rather
	173	+ * than a predicate plus a re-classification, so a caller that needs both the
	174	+ * verdict and the trigger cannot end up asking two questions with two answers.
	175	+ *
	176	+ * Every clause that can stop a refresh lives here, and that is the point: a
	177	+ * read's inline refresh, an explicit signal and a background sweep all consult
	178	+ * this, so "when do we dial an upstream" has one answer rather than one per
	179	+ * caller. A `connection.status` lifecycle (disabled, revoked) belongs here too
	180	+ * when it lands, as one more skip clause — putting it at a call site is what
	181	+ * would reintroduce the divergence this exists to prevent.
	182	+ *
	183	+ * Ordered cheapest-and-most-final first, so the reason an operator sees on the
	184	+ * span is the one they would name. A live claim observed on the row is only a
	185	+ * pre-filter: the binding decision is the compare-and-set the caller runs
	186	+ * against the database, and two readers that both see a free claim here still
	187	+ * resolve to one refresh.
	188	+ */
	189	+export type ToolSyncDecision =
	190	+ \| { readonly kind: "skip"; readonly reason: ToolSyncSkip }
	191	+ \| { readonly kind: "refresh"; readonly trigger: Exclude<ToolSyncState, "fresh"> };
	192	+
	193	+export const decideToolSync = (candidate: ToolSyncCandidate, now: number): ToolSyncDecision => {
	194	+ const state = classifyToolSync(candidate, now);
	195	+ if (state === "fresh") return { kind: "skip", reason: "fresh" };
	196	+ if (isToolSyncParked(candidate)) return { kind: "skip", reason: "parked" };
	197	+ if (isToolSyncBackedOff(candidate, now)) return { kind: "skip", reason: "backoff" };
	198	+ if (isToolSyncClaimLive(candidate, now)) return { kind: "skip", reason: "claimed" };
	199	+ return { kind: "refresh", trigger: state };
	200	+};
	201	+
	202	+/** THE eligibility predicate, for callers that only need the yes/no. */
	203	+export const isSyncEligible = (candidate: ToolSyncCandidate, now: number): boolean =>
	204	+ decideToolSync(candidate, now).kind === "refresh";
	205	+
	206	+/** The earliest next attempt after an AUTHORITATIVE listing: one freshness
	207	+ * window out, or `null` when time-based re-sync is disabled. Not a gate (see
	208	+ * {@link isToolSyncBackedOff}) — it is the horizon a scheduler sorts by. */
	209	+export const scheduleAfterSuccess = (now: number, ttlMs: number \| null): number \| null =>
	210	+ ttlMs === null ? null : now + ttlMs;
	211	+
	212	+/**
	213	+ * The earliest next attempt after a FAILED listing: `ttlMs × 2^(failures−1)`,
	214	+ * capped at {@link TOOL_SYNC_BACKOFF_CEILING_MS}, scaled by `jitter`.
	215	+ *
	216	+ * `failures` is the count INCLUDING the failure being recorded, so the first
	217	+ * failure waits one plain TTL; values below 1 are read as 1 rather than halving
	218	+ * the first delay. `jitter` belongs to the caller and is expected in
	219	+ * [{@link TOOL_SYNC_JITTER_MIN}, {@link TOOL_SYNC_JITTER_MAX}]; it is not
	220	+ * clamped here, because a call site that passes something else has a bug worth
	221	+ * seeing rather than silently correcting.
	222	+ *
	223	+ * Rounded: the result lands in a bigint column.
	224	+ */
	225	+export const scheduleAfterFailure = (
	226	+ now: number,
	227	+ ttlMs: number,
	228	+ failures: number,
	229	+ jitter: number,
	230	+): number =>
	231	+ Math.round(
	232	+ now + Math.min(TOOL_SYNC_BACKOFF_CEILING_MS, ttlMs * 2 ** (Math.max(1, failures) - 1)) * jitter,
	233	+ );