History

Accept the rare budget-yield launch reversal instead of suppressing yields

Rhys Sullivan committed 2 months ago

Commit b8edd4d1 parent 8c7833e2 files · +62 −67Browse files

Unified · Split

2 files
  1. packages/core/sdk/src/executor.test.ts +26 −29
  2. packages/core/sdk/src/executor.ts +36 −38
packages/core/sdk/src/executor.test.ts+26−29
@@ -980,31 +980,32 @@ const recordCredentialLaunch = (maxOps?: number) => · 979 unmodified lines
980 return recorder;980 return recorder;
981 });981 });
982 982
983/** Assert `start:<dominant>` is present and precedes every `start:<key>` in983/** Assert every armed read genuinely ran to completion — `start:` and
984 * `speculative`, carrying the budget and full event log into any failure. */984 * `end:` present for each key — carrying the budget and full event log into
985const expectDominantFirst = (985 * any failure. Launch ORDER is deliberately not asserted here: under an
986 * adversarial op budget the child's own cooperative yield can park the
987 * dominant read's inline launch, letting a speculative read start first.
988 * That reversal is accepted (see the launch-order comment in executor.ts —
989 * it costs no wall time on any backend, and the only way to suppress it,
990 * a fiber-lifetime `PreventSchedulerYield`, provably disables effect
991 * timeouts across provider code). What must hold at EVERY budget is that
992 * the call completes correctly and no read is lost or stranded. */
993const expectAllReadsRan = (
986 recorder: ReturnType<typeof makeReadOrderRecorder>,994 recorder: ReturnType<typeof makeReadOrderRecorder>,
987 budget: number | undefined,995 budget: number | undefined,
988 dominant: string,996 keys: readonly string[],
989 speculative: readonly string[],
990) => {997) => {
991 const dominantStart = recorder.at(`start:${dominant}`);998 const ran = keys.every(
992 const launchedFirst =999 (key) => recorder.at(`start:${key}`) >= 0 && recorder.at(`end:${key}`) >= 0,
993 dominantStart >= 0 && speculative.every((key) => dominantStart < recorder.at(`start:${key}`));1000 );
994 expect({ budget, launchedFirst, log: recorder.log() }).toMatchObject({1001 expect({ budget, ran, log: recorder.log() }).toMatchObject({ budget, ran: true });
995 budget,
996 launchedFirst: true,
997 });
998};1002};
999 1003
1000// The reproduced defect landed the run loop's op-budget yield between the1004// Sweeping low budgets lands the run loop's cooperative yield at every
1001// speculative forks and the dominant read's launch, which parked the parent's1005// position in the launch window (6 and 8 are where runtime probes reproduced
1002// continuation BEHIND the already-scheduled children and reversed the launch1006// launch-order reversals). Budgets 1 and 2 are excluded because they deadlock
1003// order. Sweeping low budgets lands that yield at every position in the1007// the effect run loop itself (a resumed fiber re-yields before evaluating a
1004// window; 6 and 8 are the two positions where runtime probes reproduced the1008// single operation), independent of this code.
1005// reversal against the pre-fix code. Budgets 1 and 2 are excluded because
1006// they deadlock the effect run loop itself (a resumed fiber re-yields before
1007// evaluating a single operation), independent of this code.
1008const ADVERSARIAL_BUDGETS: readonly number[] = [3, 4, 5, 6, 7, 8, 9, 10, 11, 12];1009const ADVERSARIAL_BUDGETS: readonly number[] = [3, 4, 5, 6, 7, 8, 9, 10, 11, 12];
1009 1010
1010describe("execute read concurrency", () => {1011describe("execute read concurrency", () => {
@@ -1057,17 +1058,13 @@ describe("execute read concurrency", () => { · 46 unmodified lines
1057 );1058 );
1058 1059
1059 it.effect(1060 it.effect(
1060 "launches the tool-row read first at every adversarial scheduler budget",1061 "completes with every pre-approval read run at every adversarial scheduler budget",
1061 () =>1062 () =>
1062 Effect.gen(function* () {1063 Effect.gen(function* () {
1063 // Only the launch-order half of the contract holds under an
1064 // adversarial budget: with a handful of ops per tick the speculative
1065 // children legitimately cannot reach their reads within the one
1066 // deferred tick the recorder grants the dominant read, so the
1067 // overlap assertions belong to the default-budget test above.
1068 for (const budget of ADVERSARIAL_BUDGETS) {1064 for (const budget of ADVERSARIAL_BUDGETS) {
1069 const recorder = yield* recordToolRowLaunch(budget);1065 const recorder = yield* recordToolRowLaunch(budget);
1070 expectDominantFirst(recorder, budget, "tool.findFirst", [1066 expectAllReadsRan(recorder, budget, [
1067 "tool.findFirst",
1071 "tool_policy.findMany",1068 "tool_policy.findMany",
1072 "connection.findFirst",1069 "connection.findFirst",
1073 ]);1070 ]);
@@ -1077,12 +1074,12 @@ describe("execute read concurrency", () => { · 3 unmodified lines
1077 );1074 );
1078 1075
1079 it.effect(1076 it.effect(
1080 "starts credential resolution first at every adversarial scheduler budget",1077 "completes with the credential and integration reads run at every adversarial budget",
1081 () =>1078 () =>
1082 Effect.gen(function* () {1079 Effect.gen(function* () {
1083 for (const budget of ADVERSARIAL_BUDGETS) {1080 for (const budget of ADVERSARIAL_BUDGETS) {
1084 const recorder = yield* recordCredentialLaunch(budget);1081 const recorder = yield* recordCredentialLaunch(budget);
1085 expectDominantFirst(recorder, budget, "credential.get", ["integration.findFirst"]);1082 expectAllReadsRan(recorder, budget, ["credential.get", "integration.findFirst"]);
1086 }1083 }
1087 }),1084 }),
1088 { timeout: 60_000 },1085 { timeout: 60_000 },
packages/core/sdk/src/executor.ts+36−38
@@ -10,7 +10,6 @@ import { · 9 unmodified lines
10 Option,10 Option,
11 Predicate,11 Predicate,
12 Ref,12 Ref,
13 Scheduler,
14 Schema,13 Schema,
15 Semaphore,14 Semaphore,
16} from "effect";15} from "effect";
@@ -5817,28 +5816,23 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea · 5800 unmodified lines
5817 // handler at construction), so an abandoned read cannot5816 // handler at construction), so an abandoned read cannot
5818 // unhandled-reject; a fiber interrupted before it ever ran issues no5817 // unhandled-reject; a fiber interrupted before it ever ran issues no
5819 // read at all.5818 // read at all.
5820 // Launch order is STRUCTURAL, not scheduling-dependent. The dominant5819 // Launch order is dominant-first by construction, best-effort under
5821 // tool-row read is forked FIRST with `startImmediately: true` under5820 // the scheduler. The dominant tool-row read is forked FIRST with
5822 // `PreventSchedulerYield`: an immediate fork evaluates the child5821 // `startImmediately: true`: an immediate fork evaluates the child
5823 // INLINE (`forkUnsafe` calls `child.evaluate` on the spot), and with5822 // INLINE (`forkUnsafe` calls `child.evaluate` on the spot), so the
5824 // the yield-prevention ref inherited at fork the child's run loop5823 // read is normally issued before the speculative forks below are
5825 // cannot insert a cooperative `yieldNow` mid-launch, so the child5824 // even scheduled (a plain fork only queues its child on the
5826 // runs synchronously until its first genuine suspension — the5825 // dispatcher for the next tick). The one exception is deliberate: if
5827 // driver-promise await, i.e. the read is ISSUED before the fork5826 // the child's own operation budget (`MaxOpsBeforeYield`) expires
5828 // returns. The speculative reads are then forked plainly, which only5827 // inside the launch window, the inline segment parks and a
5829 // schedules them on the fiber's dispatcher for the next tick. That5828 // speculative read may launch first. That reversal is materially
5830 // makes dominant-before-speculative unconditional: without the5829 // free, so it is NOT suppressed: on a synchronous driver the reads
5831 // guard, the fiber's operation budget (`MaxOpsBeforeYield`) can5830 // serialize whichever order they start (same wall time), and on an
5832 // expire anywhere in this window, and the parked continuation queues5831 // asynchronous driver every read suspends at its first await and
5833 // BEHIND already-scheduled children — plain-forking the dominant (or5832 // they overlap regardless of order. Suppressing the yield (a
5834 // forking it immediately but without `PreventSchedulerYield`) lets5833 // fiber-lifetime `PreventSchedulerYield`) is NOT an option here —
5835 // the speculative reads launch first whenever that boundary lands5834 // the ref is inherited by everything the child runs and provably
5836 // here. The cost of the ref is only that this one bounded read (one5835 // keeps effect timeouts from firing across CPU-bound stretches.
5837 // projected row) never cooperatively yields; the speculative
5838 // children are forked OUTSIDE the provide, so their scheduling is
5839 // untouched, and the dispatcher runs them whether or not this fiber
5840 // ever suspends, so a synchronous parent path delays them by one
5841 // tick but cannot starve them.
5842 const toolRowFiber = yield* Effect.forkChild(5836 const toolRowFiber = yield* Effect.forkChild(
5843 core.findFirst("tool", {5837 core.findFirst("tool", {
5844 where: (b: AnyCb) =>5838 where: (b: AnyCb) =>
@@ -5851,7 +5845,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea · 6 unmodified lines
5851 select: TOOL_INVOCATION_COLUMNS,5845 select: TOOL_INVOCATION_COLUMNS,
5852 }),5846 }),
5853 { startImmediately: true },5847 { startImmediately: true },
5854 ).pipe(Effect.provideService(Scheduler.PreventSchedulerYield, true));5848 );
5855 const policyRulesFiber = yield* Effect.forkChild(listActivePolicyRuleSet());5849 const policyRulesFiber = yield* Effect.forkChild(listActivePolicyRuleSet());
5856 const connectionRowFiber = yield* Effect.forkChild(5850 const connectionRowFiber = yield* Effect.forkChild(
5857 findConnectionRow({5851 findConnectionRow({
@@ -5957,22 +5951,26 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea · 99 unmodified lines
5957 // the two run concurrently. Both start only after5951 // the two run concurrently. Both start only after
5958 // `enforceApproval` above completes — a declined call must never5952 // `enforceApproval` above completes — a declined call must never
5959 // trigger the token refresh credential resolution can perform.5953 // trigger the token refresh credential resolution can perform.
5960 // Credential resolution is the dominant work here and must launch5954 // Credential resolution is the dominant work here and launches
5961 // first, so it uses the same structural launch-order shape as the5955 // first: an immediate fork evaluates it inline to its first
5962 // pre-approval forks above: an immediate fork under5956 // suspension before the plain integration-row fork is even
5963 // `PreventSchedulerYield` issues its first I/O inline before the5957 // scheduled (best-effort — see the launch-order comment at the
5964 // plain integration-row fork is even scheduled, no matter where5958 // pre-approval forks for why the rare budget-yield reversal is
5965 // the fiber's operation budget expires. The integration fork5959 // accepted and why suppressing it is off the table: credential
5966 // is joined after `values`, so a credential resolution failure5960 // resolution runs extension-owned provider code, and a
5967 // keeps dominating a storage failure exactly as it did when the5961 // fiber-lifetime yield guard would disable the provider-call
5968 // reads were sequential (`Fiber.join` resumes with the credential5962 // timeout across it and leak into the detached refresh fork). The
5969 // fiber's exact Exit); on that failure path the integration fork5963 // integration fork is joined after `values`, so a credential
5970 // is interrupted rather than joined, so a hung integration read5964 // resolution failure keeps dominating a storage failure exactly as
5971 // cannot gate the credential error and a failed one is deliberately5965 // it did when the reads were sequential (`Fiber.join` resumes with
5972 // abandoned, never silently dropped as an unobserved value.5966 // the credential fiber's exact Exit); on that failure path the
5967 // integration fork is interrupted rather than joined, so a hung
5968 // integration read cannot gate the credential error and a failed
5969 // one is deliberately abandoned, never silently dropped as an
5970 // unobserved value.
5973 const valuesFiber = yield* Effect.forkChild(resolveConnectionValues(connectionRow), {5971 const valuesFiber = yield* Effect.forkChild(resolveConnectionValues(connectionRow), {
5974 startImmediately: true,5972 startImmediately: true,
5975 }).pipe(Effect.provideService(Scheduler.PreventSchedulerYield, true));5973 });
5976 const integrationRowFiber = yield* Effect.forkChild(5974 const integrationRowFiber = yield* Effect.forkChild(
5977 findIntegrationRow(parsed.integration),5975 findIntegrationRow(parsed.integration),
5978 );5976 );