Fix liveness probe, --json output, and remaining unsettled state

Follow-up review of the pairing path and a second pass over seeding: - `isPersistedServerRuntimeStateLive` reported non-positive pids as live. `process.kill` reads 0 and -1 as signal-group targets rather than process lookups, so neither throws and both were caught by the "it didn't throw, so it's running" branch. A truncated or hand-edited file recording either would be believed forever — the dev candidate wins the devUrl preference, so it also outranks a genuinely live server elsewhere and mints tokens into a database nothing reads. Rejected in the schema and re-checked at the call site; the errno cast is now a guard, since an out-of-range pid throws a TypeError with no `code`. - `pairing url --json` emitted warnings on stdout ahead of the payload. The probe reads both state directories, so a corrupt file in the one *not* in use broke JSON for any consumer piping it; the quiet-logs layer was only applied around the issuing step that runs after. Verified with a crafted fixture: stdout is now valid JSON. - Turns copied in the `pending` state were left unsettled. Same defect as the `running` case fixed previously — a user message accepted before its provider turn started, which only that thread's next message would ever clear. - The source was read across a dozen implicit transactions while the installed app may be writing to it, so a project could vanish between the query that selected it and the copy that read its row. One deferred read transaction pins every query to a single snapshot. - `--from` and `--to` resolving to the same directory is now refused. Not for the data loss it appears to be (the source reads a pre-transaction snapshot, so the rows come back — verified in both journal modes), but because the event log, the cursors, and everything outside the caps do not.

Theo Browne committed 3 months ago

Commit 8a76bb81 parent 373e7918 files · +167 −26Browse files

Unified · Split

8 files

apps/server/src/cli/auth.ts +10 −1
apps/server/src/serverRuntimeState.test.ts +24 −0
apps/server/src/serverRuntimeState.ts +21 −2
scripts/dev-seed.ts +23 −2
scripts/lib/dev-seed.test.ts +12 −7
scripts/lib/dev-seed.ts +48 −11
scripts/lib/server-runtime-probe.test.ts +14 −0
scripts/lib/server-runtime-probe.ts +15 −3

apps/server/src/cli/auth.ts+10−1

@@ -210,7 +210,16 @@ const pairingUrlCommand = Command.make("url", { · 209 unmodified lines
210	210	? flags
211	211	: { ...flags, baseDir: Option.some(worktreeBaseDir) };
212	212	const config = yield* resolveCliAuthConfig(resolvedFlags, logLevel);
213		− const live = yield* findLiveServerRuntimeState(config);
	213	+ // Quieted here rather than only around the issuing layer below: probing
	214	+ // reads both state directories, and a corrupt file in the one not in
	215	+ // use logs a Warn to stdout — ahead of the payload, breaking `--json`
	216	+ // for any consumer piping it. The warning is about a file the command
	217	+ // then ignores, so under --json it is noise either way.
	218	+ const live = yield* findLiveServerRuntimeState(config).pipe(
	219	+ Effect.provide(
	220	+ Layer.succeed(References.MinimumLogLevel, flags.json ? "Error" : config.logLevel),
	221	+ ),
	222	+ );
214	223
215	224	if (Option.isNone(live)) {
216	225	return yield* new NoRunningServerError({ baseDir: config.baseDir });

apps/server/src/serverRuntimeState.test.ts+24−0

@@ -70,6 +70,30 @@ describe("serverRuntimeState", () => { · 69 unmodified lines
70	70	);
71	71	}),
72	72	);
	73	+
	74	+ // `process.kill` reads non-positive pids as signal-group targets rather
	75	+ // than process lookups — 0 is the caller's own group, -1 every permitted
	76	+ // process — so neither throws, and a probe that only catches would call
	77	+ // both "live" no matter what wrote the file.
	78	+ for (const pid of [0, -1]) {
	79	+ it.effect(`reports pid ${String(pid)} as dead rather than signalling a group`, () =>
	80	+ Effect.gen(function* () {
	81	+ assert.isFalse(
	82	+ yield* ServerRuntimeState.isPersistedServerRuntimeStateLive(stateForPid(pid)),
	83	+ );
	84	+ }),
	85	+ );
	86	+ }
	87	+
	88	+ // Out of range for a pid: process.kill throws a TypeError with no errno,
	89	+ // which the EPERM check must not assume the shape of.
	90	+ it.effect("reports an out-of-range pid as dead", () =>
	91	+ Effect.gen(function* () {
	92	+ assert.isFalse(
	93	+ yield* ServerRuntimeState.isPersistedServerRuntimeStateLive(stateForPid(2 ** 40)),
	94	+ );
	95	+ }),
	96	+ );
73	97	});
74	98
75	99	it.effect("treats a missing runtime state file as absent", () =>

apps/server/src/serverRuntimeState.ts+21−2

@@ -10,7 +10,11 @@ import { formatHostForUrl, isWildcardHost } from "./startupAccess.ts"; · 9 unmodified lines
10	10
11	11	export const PersistedServerRuntimeState = Schema.Struct({
12	12	version: Schema.Literal(1),
13		− pid: Schema.Int,
	13	+ /**
	14	+ * Positive: `process.kill` reads 0 and negatives as signal-group targets, so
	15	+ * a liveness probe on one reports "live" for a process that never existed.
	16	+ */
	17	+ pid: Schema.Int.check(Schema.isGreaterThan(0)),
14	18	host: Schema.optional(Schema.String),
15	19	port: Schema.Int,
16	20	origin: Schema.String,
@@ -107,6 +111,13 @@ export const clearPersistedServerRuntimeState = (path: string) => · 90 unmodified lines
107	111	);
108	112	});
109	113
	114	+/**
	115	+ * `process.kill` rejects an out-of-range pid with a TypeError rather than an
	116	+ * errno, so the catch below cannot assume either shape.
	117	+ */
	118	+const isErrnoException = (cause: unknown): cause is NodeJS.ErrnoException =>
	119	+ typeof cause === "object" && cause !== null && "code" in cause;
	120	+
110	121	/**
111	122	* Whether the process that wrote a runtime-state file is still alive.
112	123	*
@@ -123,12 +134,20 @@ export const isPersistedServerRuntimeStateLive = ( · 10 unmodified lines
123	134	state: PersistedServerRuntimeState,
124	135	): Effect.Effect<boolean> =>
125	136	Effect.sync(() => {
	137	+ // `process.kill` reads non-positive pids as signal-group targets rather
	138	+ // than process lookups: 0 signals the caller's own group and -1 every
	139	+ // permitted process, so neither throws and both would report "live"
	140	+ // unconditionally. A server never records one; a corrupt or truncated file
	141	+ // can, and that file would then be believed forever.
	142	+ if (!Number.isSafeInteger(state.pid) \|\| state.pid <= 0) {
	143	+ return false;
	144	+ }
126	145	try {
127	146	process.kill(state.pid, 0);
128	147	return true;
129	148	} catch (cause) {
130	149	// EPERM means the process exists but belongs to another user.
131		− return (cause as NodeJS.ErrnoException).code === "EPERM";
	150	+ return isErrnoException(cause) && cause.code === "EPERM";
132	151	}
133	152	});
134	153

scripts/dev-seed.ts+23−2

@@ -28,7 +28,13 @@ const MAX_THREAD_LIMIT = 32_766; · 27 unmodified lines
28	28	class DevSeedTargetError extends Schema.TaggedErrorClass<DevSeedTargetError>()(
29	29	"DevSeedTargetError",
30	30	{
31		− reason: Schema.Literals(["shared-home", "missing-target", "not-a-worktree", "server-running"]),
	31	+ reason: Schema.Literals([
	32	+ "shared-home",
	33	+ "missing-target",
	34	+ "not-a-worktree",
	35	+ "server-running",
	36	+ "same-source",
	37	+ ]),
32	38	detail: Schema.String,
33	39	},
34	40	) {
@@ -56,6 +62,13 @@ class DevSeedTargetError extends Schema.TaggedErrorClass<DevSeedTargetError>()( · 21 unmodified lines
56	62	"which leaves the running server's in-memory read model describing rows that no longer exist.",
57	63	"Stop it, run the seed, then start it again.",
58	64	].join("\n");
	65	+ case "same-source":
	66	+ return [
	67	+ `Refusing to seed ${this.detail} from itself.`,
	68	+ "The copy would restore what the delete removed, but the event log and the",
	69	+ "projector cursors are cleared either way — so the directory loses its history",
	70	+ "and keeps only whatever the thread and activity caps selected.",
	71	+ ].join("\n");
59	72	}
60	73	}
61	74	}
@@ -167,13 +180,21 @@ const devSeedCli = Command.make("dev-seed", { · 105 unmodified lines
167	180
168	181	// The whole point is to keep dev data off the real home; overwriting it
169	182	// here would be the exact accident this command exists to avoid.
170		− const [canonicalTarget, canonicalShared] = yield* Effect.all([
	183	+ const [canonicalTarget, canonicalSource, canonicalShared] = yield* Effect.all([
171	184	fileSystem.realPath(targetBaseDir).pipe(Effect.orElseSucceed(() => targetBaseDir)),
	185	+ fileSystem.realPath(sourceBaseDir).pipe(Effect.orElseSucceed(() => sourceBaseDir)),
172	186	fileSystem.realPath(sharedHome).pipe(Effect.orElseSucceed(() => sharedHome)),
173	187	]);
174	188	if (canonicalTarget === canonicalShared) {
175	189	return yield* new DevSeedTargetError({ reason: "shared-home", detail: targetBaseDir });
176	190	}
	191	+ // Seeding a directory from itself is not the data loss it looks like —
	192	+ // the source reads a pre-transaction snapshot, so the copy puts the rows
	193	+ // back. What does not come back is the event log and the cursors, and
	194	+ // anything outside the caps. Nobody means to ask for that.
	195	+ if (canonicalTarget === canonicalSource) {
	196	+ return yield* new DevSeedTargetError({ reason: "same-source", detail: targetBaseDir });
	197	+ }
177	198
178	199	const targetDbPath = stateDbPath(path, targetBaseDir);
179	200	if (!(yield* fileSystem.exists(targetDbPath))) {

scripts/lib/dev-seed.test.ts+12−7

@@ -101,8 +101,10 @@ function makeSource( · 100 unmodified lines
101	101	}
102	102	// The newest thread is left mid-turn, as a copy taken while an agent is
103	103	// working would be: a running turn with no completion, and a message still
104		− // streaming into it.
	104	+ // streaming into it. The one before it is left "pending" — a user message
	105	+ // accepted but whose provider turn never started.
105	106	const midFlight = index === threadCount - 1;
	107	+ const queued = index === threadCount - 2;
106	108	database
107	109	.prepare(
108	110	`INSERT INTO projection_turns (thread_id, turn_id, state, requested_at, completed_at, checkpoint_files_json)
@@ -111,9 +113,9 @@ function makeSource( · 2 unmodified lines
111	113	.run(
112	114	threadId,
113	115	`turn-${threadId}`,
114		− midFlight ? "running" : "completed",
	116	+ midFlight ? "running" : queued ? "pending" : "completed",
115	117	at,
116		− midFlight ? null : at,
	118	+ midFlight \|\| queued ? null : at,
117	119	);
118	120	database
119	121	.prepare(`INSERT INTO projection_thread_sessions VALUES (?,'running','claude',?,'boom',?)`)
@@ -297,10 +299,11 @@ describe("seedDevDatabase", () => { · 177 unmodified lines
297	299	});
298	300	});
299	301
300		− // A copied running turn has no agent left to finish it. The session override
301		− // alone does not cover this: the turn is read independently, and an unsettled
302		− // one keeps the thread spinning and its timeline unfoldable forever.
303		− it("settles turns copied mid-flight", () => {
	302	+ // A copied turn has no agent left to finish it, whether it was already
	303	+ // running or still queued behind a "pending" row. The session override alone
	304	+ // does not cover either: the turn is read independently, and an unsettled one
	305	+ // keeps the thread spinning and its timeline unfoldable forever.
	306	+ it("settles turns copied before they finished", () => {
304	307	withDatabases(({ source, target }) => {
305	308	seedDevDatabase({
306	309	sourceDbPath: source,
@@ -314,9 +317,11 @@ describe("seedDevDatabase", () => { · 7 unmodified lines
314	317	target,
315	318	"SELECT state, completed_at FROM projection_turns",
316	319	);
	320	+ // The fixture puts one running turn and one pending turn in this range.
317	321	assert.isAbove(turns.length, 0);
318	322	for (const turn of turns) {
319	323	assert.notEqual(turn.state, "running");
	324	+ assert.notEqual(turn.state, "pending");
320	325	assert.isNotNull(turn.completed_at);
321	326	}
322	327	});

scripts/lib/dev-seed.ts+48−11

@@ -216,6 +216,16 @@ function copyRows(input: { · 215 unmodified lines
216	216	return { copied, skipped };
217	217	}
218	218
	219	+/**
	220	+ * Turn states that describe work an agent still owes: "running" for a turn in
	221	+ * flight, "pending" for one whose user message was accepted but whose provider
	222	+ * turn had not started yet (`insertPendingProjectionTurn` in
	223	+ * apps/server/src/persistence/Layers/ProjectionTurns.ts). Neither can make
	224	+ * progress in a copy, and only the thread's own next message would clear a
	225	+ * pending one.
	226	+ */
	227	+const UNSETTLED_TURN_STATES = ["running", "pending"] as const;
	228	+
219	229	/**
220	230	* Settles turns copied mid-flight, matching what the server does to a turn
221	231	* whose session leaves "running": `settledTurnStateForSessionStatus` in
@@ -223,26 +233,28 @@ function copyRows(input: { · 1 unmodified line
223	233	* status the sessions are copied with to "interrupted".
224	234	*
225	235	* An UPDATE rather than a copy override, because only the rows that were
226		− * actually running should change; the rest keep their real recorded state.
	236	+ * actually unsettled should change; the rest keep their real recorded state.
227	237	*/
228		−function settleRunningTurns(
	238	+function settleUnfinishedTurns(
229	239	target: NodeSqlite.DatabaseSync,
230	240	threadIds: ReadonlyArray<string>,
231	241	): void {
232	242	if (threadIds.length === 0 \|\| !hasColumn(target, "projection_turns", "state")) {
233	243	return;
234	244	}
235		− // completed_at is what marks a turn settled alongside its state; a running
236		− // turn has none, and leaving it null keeps the thread unsettled regardless.
	245	+ // completed_at is what marks a turn settled alongside its state; an
	246	+ // unfinished turn has none, and leaving it null keeps the thread unsettled
	247	+ // regardless of the state written here.
237	248	const setCompletedAt = hasColumn(target, "projection_turns", "completed_at")
238	249	? `, completed_at = COALESCE(completed_at, requested_at)`
239	250	: "";
240	251	target
241	252	.prepare(
242	253	`UPDATE projection_turns SET state = 'interrupted'${setCompletedAt}
243		− WHERE state = 'running' AND thread_id IN (${placeholders(threadIds.length)})`,
	254	+ WHERE state IN (${placeholders(UNSETTLED_TURN_STATES.length)})
	255	+ AND thread_id IN (${placeholders(threadIds.length)})`,
244	256	)
245		− .run(...threadIds);
	257	+ .run(...UNSETTLED_TURN_STATES, ...threadIds);
246	258	}
247	259
248	260	export function seedDevDatabase(options: DevSeedOptions): DevSeedSummary {
@@ -267,6 +279,22 @@ export function seedDevDatabase(options: DevSeedOptions): DevSeedSummary { · 18 unmodified lines
267	279	);
268	280	}
269	281
	282	+ // The source is normally the live installed app's database — only the
	283	+ // target is guarded against a running server, because reading is harmless.
	284	+ // Reading it across a dozen separate implicit transactions is not: the app
	285	+ // can delete a project between the query that picks it up and the copy that
	286	+ // reads its row, leaving a copied thread whose project no longer exists (no
	287	+ // foreign keys are declared, so nothing catches it). One deferred read
	288	+ // transaction pins every query below to a single snapshot.
	289	+ let sourceTransactionOpen = false;
	290	+ try {
	291	+ source.exec("BEGIN");
	292	+ sourceTransactionOpen = true;
	293	+ } catch {
	294	+ // A concurrent writer can leave the connection unable to start one. The
	295	+ // copy is still worth doing — it just loses snapshot isolation.
	296	+ }
	297	+
270	298	try {
271	299	// Threads the user actually touched most recently. Mirrors the sidebar's own
272	300	// ordering (packages/client-runtime/src/state/threadSort.ts). Both recency
@@ -376,11 +404,11 @@ export function seedDevDatabase(options: DevSeedOptions): DevSeedSummary { · 103 unmodified lines
376	404	}),
377	405	);
378	406	// A thread copied mid-turn has no agent to finish it. The session status is
379		− // forced to "stopped" below, but the turn is read independently: a
380		− // `state = 'running'` turn keeps the thread unsettled and unfoldable
381		− // forever (deriveUnsettledTurnId in MessagesTimeline.logic.ts). Interrupted
382		− // is what the server itself settles an abandoned turn to.
383		− settleRunningTurns(target, threadIds);
	407	+ // forced to "stopped" below, but the turn is read independently: an
	408	+ // unfinished turn keeps the thread unsettled and unfoldable forever
	409	+ // (deriveUnsettledTurnId in MessagesTimeline.logic.ts). Interrupted is what
	410	+ // the server itself settles an abandoned turn to.
	411	+ settleUnfinishedTurns(target, threadIds);
384	412	const messages = record(
385	413	copyRows({
386	414	source,
@@ -475,6 +503,15 @@ export function seedDevDatabase(options: DevSeedOptions): DevSeedSummary { · 88 unmodified lines
475	503	? cause
476	504	: new DevSeedError(`could not seed the dev database: ${String(cause)}`);
477	505	} finally {
	506	+ if (sourceTransactionOpen) {
	507	+ try {
	508	+ // Read-only, so there is nothing to commit; end it before closing so
	509	+ // the snapshot is released explicitly rather than by teardown.
	510	+ source.exec("ROLLBACK");
	511	+ } catch {
	512	+ // Already ended, which is equally fine for a read-only transaction.
	513	+ }
	514	+ }
478	515	source.close();
479	516	target.close();
480	517	}

scripts/lib/server-runtime-probe.test.ts+14−0

@@ -86,6 +86,20 @@ describe("findRunningServerPid", () => { · 85 unmodified lines
86	86	}).pipe(Effect.provide(NodeServices.layer)),
87	87	);
88	88
	89	+ // `process.kill` reads these as signal-group targets, not process lookups, so
	90	+ // neither throws — a naive probe calls both "live" and blocks seeding forever
	91	+ // on a file no server ever wrote.
	92	+ for (const pid of [0, -1]) {
	93	+ it.effect(`ignores a state file recording pid ${String(pid)}`, () =>
	94	+ Effect.gen(function* () {
	95	+ const baseDir = yield* makeBaseDir();
	96	+ yield* writeRuntimeState({ baseDir, stateDir: "dev", contents: runtimeStateJson(pid) });
	97	+
	98	+ assert.isTrue(Option.isNone(yield* findRunningServerPid(baseDir)));
	99	+ }).pipe(Effect.provide(NodeServices.layer)),
	100	+ );
	101	+ }
	102	+
89	103	it.effect("reports nothing when no state files exist", () =>
90	104	Effect.gen(function* () {
91	105	const baseDir = yield* makeBaseDir();

scripts/lib/server-runtime-probe.ts+15−3

@@ -16,7 +16,8 @@ import * as Schema from "effect/Schema"; · 15 unmodified lines
16	16	const STATE_DIRS = ["dev", "userdata"] as const;
17	17
18	18	const RuntimeStateProbe = Schema.Struct({
19		− pid: Schema.Int,
	19	+ /** Positive, for the reason `isPidLive` explains. */
	20	+ pid: Schema.Int.check(Schema.isGreaterThan(0)),
20	21	});
21	22
22	23	const decodeRuntimeStateProbe = Schema.decodeUnknownEffect(
@@ -34,12 +35,23 @@ const decodeRuntimeStateProbe = Schema.decodeUnknownEffect( · 11 unmodified lines
34	35	* case is being told to stop a server that is already stopped.
35	36	*/
36	37	const isPidLive = (pid: number): boolean => {
	38	+ // `process.kill` reads non-positive pids as signal-group targets rather than
	39	+ // process lookups — 0 signals the caller's own group, -1 every permitted
	40	+ // process — so neither throws and both would report "live" unconditionally.
	41	+ // The schema rejects those, and this repeats the check so the invariant
	42	+ // travels with the code that depends on it.
	43	+ if (!Number.isSafeInteger(pid) \|\| pid <= 0) {
	44	+ return false;
	45	+ }
37	46	try {
38	47	process.kill(pid, 0);
39	48	return true;
40	49	} catch (cause) {
41		− // EPERM means the process exists but belongs to another user.
42		− return (cause as NodeJS.ErrnoException).code === "EPERM";
	50	+ // EPERM means the process exists but belongs to another user. An
	51	+ // out-of-range pid throws a TypeError instead, which has no errno.
	52	+ return typeof cause === "object" && cause !== null && "code" in cause
	53	+ ? cause.code === "EPERM"
	54	+ : false;
43	55	}
44	56	};
45	57