// --------------------------------------------------------------------------- // Tool-catalog sync lifecycle — the pure state model behind every decision the // executor makes about re-listing a connection's tools. // // No storage, no clock, no randomness: the connection's columns arrive as a // value, `now` and the backoff jitter arrive as arguments, and every function // here is total. That is what makes the ladder, the cap and the eligibility // rules testable directly rather than through a database and a live upstream, // and it is what will let PR 3's background scheduler reuse the same rules // without reimplementing them against a different clock. // --------------------------------------------------------------------------- /** Why a catalog listing could not be completed, as a CLOSED set persisted on * `connection.tools_sync_error_kind`. * * Structural on purpose: the previous mechanism was a `"Tool sync failing: "` * prefix on the human-readable health detail, which meant every consumer that * wanted to know "is this connection's catalog broken, and how" had to match a * display string. These four values are the whole vocabulary: * * - `auth` the credential is rejected or absent. Retrying the same grant * cannot change the verdict, so it PARKS the connection (see * {@link isToolSyncParked}) until a human re-authorizes. * - `unreachable` the upstream did not answer (DNS, connect, timeout). * - `protocol` the upstream answered but the listing could not be read. * - `config` the integration's own configuration cannot produce a listing * (unparseable spec, disabled transport, missing endpoint). */ export type ToolSyncErrorKind = "auth" | "unreachable" | "protocol" | "config"; export const TOOL_SYNC_ERROR_KINDS = [ "auth", "unreachable", "protocol", "config", ] as const satisfies readonly ToolSyncErrorKind[]; /** Narrow a persisted `tools_sync_error_kind` column back to the closed set. * The column is plain text, so a value written by a newer build (or edited by * hand) has to be parsed rather than trusted. */ export const isToolSyncErrorKind = (value: unknown): value is ToolSyncErrorKind => typeof value === "string" && (TOOL_SYNC_ERROR_KINDS as readonly string[]).includes(value); /** * What a connection's persisted catalog is, relative to now. * * Every non-`fresh` value is exactly the reason a read has to re-list, so this * doubles as the refresh trigger reported on the `executor.tools.sync` span. * * `cold` and `stale_marked` were one value before the `tools_stale_token` * column existed: `connections.markToolsStale` cleared `tools_synced_at`, which * made a connection that had never synced indistinguishable from one whose * catalog was invalidated mid-invocation, and destroyed the last-verified * timestamp in the process. */ export type ToolSyncState = "fresh" | "cold" | "stale_marked" | "config_revised" | "expired"; /** Why an otherwise-due connection is not being refreshed right now. Enumerable * so the read's span can count each reason. */ export type ToolSyncSkip = "fresh" | "parked" | "backoff" | "claimed"; /** * A connection's sync lifecycle columns, plus the two facts that live on its * integration and plugin (`config_revised_at`, whether the plugin lists a live * remote catalog) and the executor's freshness window. * * Decoded from the row by the caller: bigint columns arrive as `bigint | null` * from the driver, and the text `tools_sync_error_kind` has to pass * {@link isToolSyncErrorKind} before it gets here. */ export interface ToolSyncCandidate { /** Epoch ms of the last AUTHORITATIVE listing. Never stamped by a failed or * incomplete one — a month-dead server must not read as "synced 30s ago". */ readonly toolsSyncedAt: number | null; /** The outstanding drift mark, or null when there is none. An opaque token * (see `connection.tools_stale_token`) — its presence is the whole signal * and its value is only ever compared for equality, by the listing that * clears it. */ readonly toolsStaleToken: string | null; /** The nonce of the refresh attempt currently holding the write lease. */ readonly toolsSyncClaimId: string | null; /** Epoch ms the claim was taken, against which the lease expires. */ readonly toolsSyncClaimAt: number | null; /** CONSECUTIVE failed listings; zeroed by an authoritative one. */ readonly toolsSyncFailures: number; /** Epoch ms of the earliest next attempt. */ readonly toolsSyncRetryAt: number | null; readonly toolsSyncErrorKind: ToolSyncErrorKind | null; /** The integration's `config_revised_at`: its last tool-affecting config * change. */ readonly configRevisedAt: number | null; /** The plugin lists a live remote catalog (an MCP server, whose tool set can * change with no executor-visible signal), so time alone can expire it. */ readonly remoteToolCatalog: boolean; /** The freshness window; `null` disables time-based expiry entirely. */ readonly ttlMs: number | null; } /** * How long a refresh claim stays valid. * * A lease rather than a lock: the claimant can be a Workers isolate that is * evicted mid-handshake, and nothing would ever release its claim. Long enough * to cover a slow upstream listing (discovery is bounded well under this), * short enough that a lost claimant costs one wasted freshness window. */ export const TOOL_SYNC_CLAIM_LEASE_MS = 60_000; /** The longest a failing connection is left alone. Past this the ladder stops * doubling: a server that has been dead for a week still gets probed a few * times a day, which is what makes recovery automatic. */ export const TOOL_SYNC_BACKOFF_CEILING_MS = 6 * 60 * 60 * 1000; /** Bounds of the multiplicative jitter applied to every backoff delay, so a * fleet that lost the same upstream at the same moment does not re-dial it in * lockstep. The factor itself is supplied by the caller — this module owns no * randomness. */ export const TOOL_SYNC_JITTER_MIN = 0.8; export const TOOL_SYNC_JITTER_MAX = 1.2; /** * Classify a connection's catalog. Total, and ordered by authority: an explicit * drift signal outranks a config revision, which outranks the clock. * * A drift mark is read as PRESENCE, never as an instant: `tools_stale_token` is * an opaque token, and the listing that resolves it clears it by * compare-and-set on the token it observed. Comparing a drift TIMESTAMP against * `tools_synced_at` instead is what made the two stamps landing in one * millisecond ambiguous, and left the column set forever on every connection * that had ever drifted. * * The config-revision comparison stays inclusive of the stamp's own millisecond * (`syncedAt <= configRevisedAt`). `Date.now()` has millisecond granularity and * `tools_synced_at` records when the listing STARTED, so a revision landing in * that millisecond describes a config the listing cannot have read. Swallowing * it would leave the connection serving a catalog that predates the change with * no TTL backstop — a plugin without a remote catalog never reaches the * `expired` branch below. */ export const classifyToolSync = (candidate: ToolSyncCandidate, now: number): ToolSyncState => { const syncedAt = candidate.toolsSyncedAt; if (candidate.toolsStaleToken !== null) return "stale_marked"; if (syncedAt === null) return "cold"; if (candidate.configRevisedAt !== null && syncedAt <= candidate.configRevisedAt) { return "config_revised"; } if (candidate.remoteToolCatalog && candidate.ttlMs !== null && syncedAt < now - candidate.ttlMs) { return "expired"; } return "fresh"; }; /** Whether another attempt currently holds the write lease. A claim with no * `claim_at` is treated as dead rather than eternal, so a half-written row can * never wedge a connection out of every future refresh. */ export const isToolSyncClaimLive = (candidate: ToolSyncCandidate, now: number): boolean => candidate.toolsSyncClaimId !== null && candidate.toolsSyncClaimAt !== null && now - candidate.toolsSyncClaimAt < TOOL_SYNC_CLAIM_LEASE_MS; /** * Whether the failure ladder is still holding this connection off. * * Gated on `failures > 0` deliberately. `tools_sync_retry_at` is also written * after a SUCCESS, where it records the TTL horizon so a scheduler can order * work by one column; treating that as a gate would make a `list_changed` * notification wait out the freshness window before it could heal the catalog. * An event-driven drift re-lists immediately; only a failing one waits. */ export const isToolSyncBackedOff = (candidate: ToolSyncCandidate, now: number): boolean => candidate.toolsSyncFailures > 0 && candidate.toolsSyncRetryAt !== null && now < candidate.toolsSyncRetryAt; /** Whether the connection is parked: its credential is rejected, and no amount * of retrying the same grant will change that. Cleared by the human-initiated * paths that can actually fix it (explicit refresh, connection update, an * OAuth re-mint, a healthy probe, a fresh drift signal). {@link decideToolSync} * applies it to `expired` alone — every other refresh trigger outranks it. */ export const isToolSyncParked = (candidate: ToolSyncCandidate): boolean => candidate.toolsSyncErrorKind === "auth"; /** * Refresh this connection, or pass it over and say why. ONE decision rather * than a predicate plus a re-classification, so a caller that needs both the * verdict and the trigger cannot end up asking two questions with two answers. * * Every clause that can stop a refresh lives here, and that is the point: a * read's inline refresh, an explicit signal and a background sweep all consult * this, so "when do we dial an upstream" has one answer rather than one per * caller. A `connection.status` lifecycle (disabled, revoked) belongs here too * when it lands, as one more skip clause — putting it at a call site is what * would reintroduce the divergence this exists to prevent. * * Ordered cheapest-and-most-final first, so the reason an operator sees on the * span is the one they would name: fresh, then parked, then backoff, then * claimed. A live claim observed on the row is only a pre-filter: the binding * decision is the compare-and-set the caller runs against the database, and two * readers that both see a free claim here still resolve to one refresh. * * The park gates `expired` and NOTHING else. `expired` is the only trigger that * is purely the clock asking a connection which already has a working catalog * to re-verify it, and re-dialing a rejected credential to re-verify a catalog * we can still serve is exactly the waste the park exists to stop. * * `stale_marked` and `config_revised` are somebody telling us the world * changed. A config revision in particular is how an `auth` verdict caused by * integration configuration (the key in the wrong place, the wrong token * endpoint) gets FIXED, and `integrations.update` is the one invalidation path * that does not clear the ladder, so swallowing it would leave the repaired * connection parked forever. * * `cold` is deliberately NOT parked, and that is a change from parking every * clock-driven trigger. A parked connection that has never synced has no * catalog to serve, so parking it is not "keep serving what we have", it is * "serve nothing, forever, until a human notices". The retry ladder already * caps a never-synced auth failure at roughly four dials a day, and letting the * clock drive it back is what restores automatic recovery when the credential * is repaired upstream — where nothing in this system observes the repair. */ export type ToolSyncDecision = | { readonly kind: "skip"; readonly reason: ToolSyncSkip } | { readonly kind: "refresh"; readonly trigger: Exclude }; export const decideToolSync = (candidate: ToolSyncCandidate, now: number): ToolSyncDecision => { const state = classifyToolSync(candidate, now); if (state === "fresh") return { kind: "skip", reason: "fresh" }; if (state === "expired" && isToolSyncParked(candidate)) return { kind: "skip", reason: "parked" }; if (isToolSyncBackedOff(candidate, now)) return { kind: "skip", reason: "backoff" }; if (isToolSyncClaimLive(candidate, now)) return { kind: "skip", reason: "claimed" }; return { kind: "refresh", trigger: state }; }; /** THE eligibility predicate, for callers that only need the yes/no. */ export const isSyncEligible = (candidate: ToolSyncCandidate, now: number): boolean => decideToolSync(candidate, now).kind === "refresh"; /** The earliest next attempt after an AUTHORITATIVE listing: one freshness * window out, or `null` when time-based re-sync is disabled. Not a gate (see * {@link isToolSyncBackedOff}) — it is the horizon a scheduler sorts by. */ export const scheduleAfterSuccess = (now: number, ttlMs: number | null): number | null => ttlMs === null ? null : now + ttlMs; /** * The earliest next attempt after a FAILED listing: `ttlMs × 2^(failures−1)`, * capped at {@link TOOL_SYNC_BACKOFF_CEILING_MS}, scaled by `jitter`. * * `failures` is the count INCLUDING the failure being recorded, so it is `>= 1` * and the first failure waits one plain TTL. Neither argument is clamped: a * call site that passes the PRIOR count is indistinguishable from a correct one * at the first rung and then runs the whole ladder one step short, permanently * and invisibly, which is a bug worth seeing rather than laundering into a * plausible delay. `jitter` belongs to the caller on the same terms and is * expected in [{@link TOOL_SYNC_JITTER_MIN}, {@link TOOL_SYNC_JITTER_MAX}]. * * Rounded: the result lands in a bigint column. */ export const scheduleAfterFailure = ( now: number, ttlMs: number, failures: number, jitter: number, ): number => Math.round(now + Math.min(TOOL_SYNC_BACKOFF_CEILING_MS, ttlMs * 2 ** (failures - 1)) * jitter);