493 lines · 17.7 KB · TypeScript
RawDownload
import { describe, expect, it } from "@effect/vitest"; import {  TOOL_SYNC_BACKOFF_CEILING_MS,  TOOL_SYNC_CLAIM_LEASE_MS,  TOOL_SYNC_ERROR_KINDS,  TOOL_SYNC_JITTER_MAX,  TOOL_SYNC_JITTER_MIN,  classifyToolSync,  decideToolSync,  isSyncEligible,  isToolSyncBackedOff,  isToolSyncClaimLive,  isToolSyncErrorKind,  isToolSyncParked,  scheduleAfterFailure,  scheduleAfterSuccess,  type ToolSyncCandidate,} from "./tool-sync-schedule"; // The catalog sync lifecycle is a pure decision function, and these are the// decisions it exists to make: what a persisted catalog IS, whether anyone is// allowed to re-list it right now, and how long a failing connection is left// alone. Testing it here rather than only through the executor is what keeps// the ladder, the cap and the park rule honest without a database and a live// upstream in the loop. const NOW = 1_700_000_000_000;const TTL = 15 * 60 * 1000; /** A drift mark, as `connections.markToolsStale` writes it. Opaque: nothing *  here may read structure out of it, and nothing may order two of them. */const STALE_TOKEN = "stale_m3k1x9q2"; /** A row every case overrides into the state it is about. Neutral on purpose: *  naming it for one of the states under test (it started life as `fresh`) *  makes `classifyToolSync(fresh({ toolsSyncedAt: null }))` read as a *  contradiction with the assertion beside it. */const candidate = (overrides: Partial<ToolSyncCandidate> = {}): ToolSyncCandidate => ({  toolsSyncedAt: NOW - 1000,  toolsStaleToken: null,  toolsSyncClaimId: null,  toolsSyncClaimAt: null,  toolsSyncFailures: 0,  toolsSyncRetryAt: null,  toolsSyncErrorKind: null,  configRevisedAt: null,  remoteToolCatalog: true,  ttlMs: TTL,  ...overrides,}); describe("isToolSyncErrorKind", () => {  it("narrows exactly the kinds the closed set declares", () => {    // Pinned as a literal, not derived: the set is closed because    // `isToolSyncParked`'s auth-only rule and the persisted-column parse depend    // on it, so a fifth kind has to be a deliberate edit here rather than a    // silent widening.    expect([...TOOL_SYNC_ERROR_KINDS]).toEqual(["auth", "unreachable", "protocol", "config"]);    for (const kind of TOOL_SYNC_ERROR_KINDS) {      expect(isToolSyncErrorKind(kind)).toBe(true);    }  });   it("rejects anything else a text column could hold", () => {    for (const value of ["", "AUTH", "authentication", null, undefined, 401, {}]) {      expect(isToolSyncErrorKind(value)).toBe(false);    }  });}); describe("classifyToolSync", () => {  it("calls a never-synced, never-marked connection cold", () => {    expect(classifyToolSync(candidate({ toolsSyncedAt: null }), NOW)).toBe("cold");  });   it("calls a connection with a drift mark stale_marked, keeping its stamp", () => {    expect(      classifyToolSync(        candidate({ toolsSyncedAt: NOW - 60_000, toolsStaleToken: STALE_TOKEN }),        NOW,      ),    ).toBe("stale_marked");  });   it("reads the drift mark as presence, never against the stamp", () => {    // The whole point of the token encoding. `tools_stale_token` answers "was    // this catalog invalidated since the listing now finishing began", which is    // a version question; the earlier encoding asked it of a wall clock, and    // `Date.now()` has millisecond granularity, so a mark and a stamp landing in    // one millisecond tied. No stamp, however far ahead of the mark, resolves    // it — only the compare-and-set that nulls the column does.    for (const toolsSyncedAt of [null, 0, NOW - 60_000, NOW, NOW + 60_000]) {      expect(        classifyToolSync(candidate({ toolsSyncedAt, toolsStaleToken: STALE_TOKEN }), NOW),      ).toBe("stale_marked");    }  });   it("leaves a connection whose drift mark was cleared resolved", () => {    // An authoritative listing clears the token it observed, and a cleared    // column is the only thing that settles a drift. This is also what keeps    // the read's scan at zero rows in steady state: nothing ever cleared the    // old timestamp column, so every connection that had ever drifted was    // re-selected on every read, forever.    expect(      classifyToolSync(candidate({ toolsSyncedAt: NOW - 1000, toolsStaleToken: null }), NOW),    ).toBe("fresh");  });   it("calls a never-synced connection carrying a drift mark stale_marked, not cold", () => {    expect(      classifyToolSync(candidate({ toolsSyncedAt: null, toolsStaleToken: STALE_TOKEN }), NOW),    ).toBe("stale_marked");  });   it("calls a catalog older than its integration's config revision config_revised", () => {    expect(      classifyToolSync(        candidate({ toolsSyncedAt: NOW - 60_000, configRevisedAt: NOW - 30_000 }),        NOW,      ),    ).toBe("config_revised");  });   it("still sees a config revision that landed in the same millisecond as the stamp", () => {    // Same rule as the drift mark, and for a stronger reason: `tools_synced_at`    // is stamped when the listing STARTED, so a revision inside that    // millisecond describes a config the listing cannot have read. A plugin    // without a remote catalog has no TTL backstop to catch it later.    expect(classifyToolSync(candidate({ toolsSyncedAt: NOW, configRevisedAt: NOW }), NOW)).toBe(      "config_revised",    );  });   it("leaves a catalog newer than the config revision alone", () => {    expect(      classifyToolSync(        candidate({ toolsSyncedAt: NOW - 30_000, configRevisedAt: NOW - 60_000 }),        NOW,      ),    ).toBe("fresh");  });   it("expires a remote catalog on the far side of the TTL only", () => {    // Both sides of the boundary: flipping the comparison, or shifting the    // subtraction, changes when every remote catalog re-dials its upstream.    expect(classifyToolSync(candidate({ toolsSyncedAt: NOW - TTL }), NOW)).toBe("fresh");    expect(classifyToolSync(candidate({ toolsSyncedAt: NOW - TTL - 1 }), NOW)).toBe("expired");  });   it("never expires a catalog the plugin does not list remotely", () => {    // An OpenAPI catalog is derived from a stored spec: time alone cannot make    // it wrong, and re-listing it on a timer is pure waste.    expect(      classifyToolSync(candidate({ toolsSyncedAt: NOW - TTL - 1, remoteToolCatalog: false }), NOW),    ).toBe("fresh");  });   it("never expires anything when the TTL is disabled", () => {    expect(classifyToolSync(candidate({ toolsSyncedAt: 0, ttlMs: null }), NOW)).toBe("fresh");  });   it("ranks an explicit drift signal above a config revision and the clock", () => {    expect(      classifyToolSync(        candidate({          toolsSyncedAt: NOW - TTL - 1,          toolsStaleToken: STALE_TOKEN,          configRevisedAt: NOW - 1,        }),        NOW,      ),    ).toBe("stale_marked");  });   it("ranks a config revision above the clock", () => {    expect(      classifyToolSync(candidate({ toolsSyncedAt: NOW - TTL - 1, configRevisedAt: NOW - 1 }), NOW),    ).toBe("config_revised");  });}); describe("isToolSyncClaimLive", () => {  it("is false with no claim", () => {    expect(isToolSyncClaimLive(candidate(), NOW)).toBe(false);  });   it("is true inside the lease", () => {    expect(      isToolSyncClaimLive(        candidate({          toolsSyncClaimId: "sync_abc",          toolsSyncClaimAt: NOW - TOOL_SYNC_CLAIM_LEASE_MS + 1,        }),        NOW,      ),    ).toBe(true);  });   it("is false once the lease has run out", () => {    expect(      isToolSyncClaimLive(        candidate({          toolsSyncClaimId: "sync_abc",          toolsSyncClaimAt: NOW - TOOL_SYNC_CLAIM_LEASE_MS,        }),        NOW,      ),    ).toBe(false);  });   it("treats a claim with no timestamp as dead rather than eternal", () => {    // Otherwise a half-written row would lock a connection out of every future    // refresh, forever, with nothing to expire it.    expect(      isToolSyncClaimLive(candidate({ toolsSyncClaimId: "sync_abc", toolsSyncClaimAt: null }), NOW),    ).toBe(false);  });}); describe("isToolSyncBackedOff", () => {  it("holds a failing connection off until its retry instant", () => {    const backedOff = candidate({ toolsSyncFailures: 2, toolsSyncRetryAt: NOW + 1 });    expect(isToolSyncBackedOff(backedOff, NOW)).toBe(true);    expect(isToolSyncBackedOff(backedOff, NOW + 1)).toBe(false);  });   it("ignores the retry instant a SUCCESS wrote", () => {    // After a success `tools_sync_retry_at` carries the freshness horizon, not    // a gate: a drift signal arriving inside the window must heal the catalog    // immediately instead of waiting the window out.    const drifted = candidate({      toolsSyncFailures: 0,      toolsSyncRetryAt: NOW + TTL,      toolsStaleToken: STALE_TOKEN,    });    expect(isToolSyncBackedOff(drifted, NOW)).toBe(false);    expect(isSyncEligible(drifted, NOW)).toBe(true);  });}); describe("isToolSyncParked", () => {  it("parks only on an auth verdict", () => {    expect(isToolSyncParked(candidate({ toolsSyncErrorKind: "auth" }))).toBe(true);    for (const kind of ["unreachable", "protocol", "config", null] as const) {      expect(isToolSyncParked(candidate({ toolsSyncErrorKind: kind }))).toBe(false);    }  });}); describe("decideToolSync", () => {  it("passes a due connection with a clear ledger, reporting what made it due", () => {    expect(decideToolSync(candidate({ toolsSyncedAt: null }), NOW)).toEqual({      kind: "refresh",      trigger: "cold",    });  });   it("skips a fresh connection", () => {    expect(decideToolSync(candidate(), NOW)).toEqual({ kind: "skip", reason: "fresh" });  });   it("skips a parked connection however far past its freshness window it is", () => {    expect(      decideToolSync(        candidate({          toolsSyncedAt: NOW - 30 * TTL,          toolsSyncFailures: 9,          toolsSyncErrorKind: "auth",          toolsSyncRetryAt: NOW - 1,        }),        NOW,      ),    ).toEqual({ kind: "skip", reason: "parked" });  });   it("skips a connection inside its backoff window", () => {    expect(      decideToolSync(        candidate({          toolsSyncedAt: null,          toolsSyncFailures: 1,          toolsSyncErrorKind: "unreachable",          toolsSyncRetryAt: NOW + 1,        }),        NOW,      ),    ).toEqual({ kind: "skip", reason: "backoff" });  });   it("skips a connection another attempt is already holding", () => {    expect(      decideToolSync(        candidate({ toolsSyncedAt: null, toolsSyncClaimId: "sync_abc", toolsSyncClaimAt: NOW }),        NOW,      ),    ).toEqual({ kind: "skip", reason: "claimed" });  });   it("re-admits a connection whose claimant died", () => {    expect(      decideToolSync(        candidate({          toolsSyncedAt: null,          toolsSyncClaimId: "sync_abc",          toolsSyncClaimAt: NOW - TOOL_SYNC_CLAIM_LEASE_MS,        }),        NOW,      ),    ).toEqual({ kind: "refresh", trigger: "cold" });  });   // The skip reason is not cosmetic: the read fans it into three separate  // operator counters, so a clause reordering silently misattributes every sync  // skip. Each case below satisfies SEVERAL clauses at once, which is the only  // way the order itself is what the assertion pins.  describe("precedence", () => {    it("reports fresh over an auth verdict left on the row", () => {      // A parked connection that has since been re-listed is FRESH. Ranking the      // park first would count every healthy connection carrying a stale error      // kind as parked, on every read.      expect(decideToolSync(candidate({ toolsSyncErrorKind: "auth" }), NOW)).toEqual({        kind: "skip",        reason: "fresh",      });    });     it("reports parked over a backoff window that is still open", () => {      expect(        decideToolSync(          candidate({            toolsSyncedAt: NOW - TTL - 1,            toolsSyncErrorKind: "auth",            toolsSyncFailures: 3,            toolsSyncRetryAt: NOW + TTL,          }),          NOW,        ),      ).toEqual({ kind: "skip", reason: "parked" });    });     it("reports backoff, not parked, for a COLD connection with an auth verdict", () => {      // The park does not apply to `cold`, so the ladder is what holds this      // connection off — and the reason an operator sees has to say so, because      // the read fans the two into different counters and they mean opposite      // things: `skipped_parked` is waste permanently avoided, `skipped_backoff`      // is a retry that will happen.      expect(        decideToolSync(          candidate({            toolsSyncedAt: null,            toolsSyncErrorKind: "auth",            toolsSyncFailures: 3,            toolsSyncRetryAt: NOW + TTL,          }),          NOW,        ),      ).toEqual({ kind: "skip", reason: "backoff" });    });     it("reports backoff over a live claim", () => {      expect(        decideToolSync(          candidate({            toolsSyncedAt: null,            toolsSyncFailures: 2,            toolsSyncRetryAt: NOW + 1,            toolsSyncClaimId: "sync_abc",            toolsSyncClaimAt: NOW,          }),          NOW,        ),      ).toEqual({ kind: "skip", reason: "backoff" });    });  });   // The park answers exactly one question: "is re-VERIFYING a catalog we can  // still serve worth dialing a credential that was refused". Every other  // trigger outranks it.  describe("park gates expired and nothing else", () => {    it("refreshes a parked connection whose integration config was revised", () => {      // This is the repair path: an `auth` verdict caused by integration      // configuration is fixed by editing that configuration, and      // `integrations.update` is the one invalidation that does not clear the      // ladder. Swallowing it leaves the fixed connection parked forever.      expect(        decideToolSync(          candidate({            toolsSyncedAt: NOW - 60_000,            configRevisedAt: NOW - 1,            toolsSyncErrorKind: "auth",          }),          NOW,        ),      ).toEqual({ kind: "refresh", trigger: "config_revised" });    });     it("refreshes a parked connection that was marked stale", () => {      expect(        decideToolSync(          candidate({            toolsSyncedAt: NOW - 60_000,            toolsStaleToken: STALE_TOKEN,            toolsSyncErrorKind: "auth",          }),          NOW,        ),      ).toEqual({ kind: "refresh", trigger: "stale_marked" });    });     it("refreshes a parked connection that has never synced at all", () => {      // A parked `cold` connection has NO catalog to serve, so parking it is      // not "keep serving what we have", it is "serve nothing until a human      // notices". The ladder still caps the cost at a few dials a day, and it      // is what lets a credential repaired upstream — where nothing in this      // system observes the repair — recover on its own.      expect(        decideToolSync(candidate({ toolsSyncedAt: null, toolsSyncErrorKind: "auth" }), NOW),      ).toEqual({ kind: "refresh", trigger: "cold" });    });     it("still parks when the clock is only asking for a re-verification", () => {      expect(        decideToolSync(          candidate({ toolsSyncedAt: NOW - TTL - 1, toolsSyncErrorKind: "auth" }),          NOW,        ),      ).toEqual({ kind: "skip", reason: "parked" });    });  });}); describe("isSyncEligible", () => {  it("projects the decision to a boolean", () => {    expect(isSyncEligible(candidate({ toolsSyncedAt: null }), NOW)).toBe(true);    expect(isSyncEligible(candidate(), NOW)).toBe(false);  });}); describe("scheduleAfterSuccess", () => {  it("puts the next attempt one freshness window out", () => {    expect(scheduleAfterSuccess(NOW, TTL)).toBe(NOW + TTL);  });   it("schedules nothing when time-based re-sync is disabled", () => {    expect(scheduleAfterSuccess(NOW, null)).toBe(null);  });}); describe("scheduleAfterFailure", () => {  const delay = (failures: number, jitter = 1, ttlMs = TTL) =>    scheduleAfterFailure(NOW, ttlMs, failures, jitter) - NOW;   it("doubles the wait with every consecutive failure", () => {    // `failures` is the count INCLUDING the failure being recorded, so 1 is the    // first rung. Out-of-contract counts are not clamped — a caller passing the    // prior count would run the whole ladder one step short, and that is a bug    // to surface rather than launder.    expect(delay(1)).toBe(TTL);    expect(delay(2)).toBe(2 * TTL);    expect(delay(3)).toBe(4 * TTL);    expect(delay(4)).toBe(8 * TTL);  });   it("caps the ladder rather than growing without bound", () => {    // A server dead for a week is still probed a few times a day, which is what    // makes recovery automatic.    expect(delay(20)).toBe(TOOL_SYNC_BACKOFF_CEILING_MS);    expect(delay(1000)).toBe(TOOL_SYNC_BACKOFF_CEILING_MS);  });   it("stays within the jitter bounds at every rung", () => {    for (let failures = 1; failures <= 12; failures += 1) {      const base = Math.min(TOOL_SYNC_BACKOFF_CEILING_MS, TTL * 2 ** (failures - 1));      expect(delay(failures, TOOL_SYNC_JITTER_MIN)).toBe(Math.round(base * TOOL_SYNC_JITTER_MIN));      expect(delay(failures, TOOL_SYNC_JITTER_MAX)).toBe(Math.round(base * TOOL_SYNC_JITTER_MAX));      for (const jitter of [TOOL_SYNC_JITTER_MIN, 0.93, 1.07, TOOL_SYNC_JITTER_MAX]) {        const value = delay(failures, jitter);        expect(value).toBeGreaterThanOrEqual(Math.round(base * TOOL_SYNC_JITTER_MIN));        expect(value).toBeLessThanOrEqual(Math.round(base * TOOL_SYNC_JITTER_MAX));      }    }  });   it("returns a whole number of milliseconds for the bigint column", () => {    expect(Number.isInteger(scheduleAfterFailure(NOW, TTL, 3, 0.8137))).toBe(true);  });   it("scales the ladder to whatever base it is given", () => {    expect(delay(5, 1, 1000)).toBe(16_000);  });});

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t