fix(server): validate update rollback boundaries

Theo Browne committed 3 months ago

Commit cd25a3d1 parent c07288814 files · +499 −48Browse files

Unified · Split

14 files

apps/server/src/cli/service.ts +2 −2
apps/server/src/cloud/selfUpdate.test.ts +1 −0
apps/server/src/cloud/selfUpdate.ts +1 −0
apps/server/src/cloud/systemdSelfUpdate.acceptance.test.ts +237 −0
apps/server/src/cloud/systemdSelfUpdate.test.ts +56 −4
apps/server/src/cloud/systemdSelfUpdate.ts +8 −7
apps/server/src/http.ts +18 −7
apps/server/src/persistence/Migrations.test.ts +51 −1
apps/server/src/persistence/Migrations.ts +58 −18
apps/server/src/server.test.ts +25 −0
apps/server/src/serverRuntimeStartup.ts +5 −0
docs/internals/server-updates.md +9 −4
docs/operations/release.md +17 −0
packages/contracts/src/environmentHttp.ts +11 −5

@@ -9,7 +9,7 @@ import packageJson from "../../package.json" with { type: "json" }; · 8 unmodified lines
9	9	import * as BootService from "../cloud/bootService.ts";
10	10	import { applySystemdSelfUpdatePlan } from "../cloud/systemdSelfUpdate.ts";
11	11	import type * as ServerConfig from "../config.ts";
12		−import { validateMigrationIdentities } from "../persistence/Migrations.ts";
	12	+import { validateAutomaticUpdateMigrationFrontier } from "../persistence/Migrations.ts";
13	13	import * as ProcessRunner from "../processRunner.ts";
14	14	import { projectLocationFlags, resolveCliAuthConfig } from "./config.ts";
15	15
@@ -169,7 +169,7 @@ const serviceValidateUpdateCommand = Command.make("_validate-update", { · 153 unmodified lines
169	169	const NodeSqliteClient = yield* Effect.promise(
170	170	() => import("../persistence/NodeSqliteClient.ts"),
171	171	);
172		− yield* validateMigrationIdentities().pipe(
	172	+ yield* validateAutomaticUpdateMigrationFrontier().pipe(
173	173	Effect.scoped,
174	174	Effect.provide(NodeSqliteClient.layer({ filename: database, readonly: true })),
175	175	);

apps/server/src/cloud/selfUpdate.test.ts+1−0

@@ -535,6 +535,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { · 534 unmodified lines
535	535	Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(SystemdSelfUpdatePlan))),
536	536	);
537	537	assert.equal(plan.previousUnit, previousUnit);
	538	+ assert.equal(plan.serviceUnit, BOOT_SERVICE_UNIT_FILE);
538	539	assert.include(plan.nextUnit, `ExecStart=${NODE_PATH} ${pinnedEntry} serve`);
539	540	assert.equal(
540	541	plan.runtimeStatePath,

apps/server/src/cloud/selfUpdate.ts+1−0

@@ -394,6 +394,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option · 393 unmodified lines
394	394	fromVersion: packageJson.version,
395	395	targetVersion,
396	396	currentPid: process.pid,
	397	+ serviceUnit: BOOT_SERVICE_UNIT_FILE,
397	398	unitPath,
398	399	previousUnit,
399	400	nextUnit,

apps/server/src/cloud/systemdSelfUpdate.acceptance.test.ts+237−0

@@ -0,0 +1,237 @@
	1	+// @effect-diagnostics nodeBuiltinImport:off
	2	+/**
	3	+ * Opt-in acceptance coverage for the real systemd handoff and rollback path.
	4	+ * Every unit and file is uniquely named; this must never target t3code.service.
	5	+ *
	6	+ * Run on Linux with a user manager:
	7	+ * T3_SYSTEMD_SELF_UPDATE_ACCEPTANCE=1 vp test run \
	8	+ * apps/server/src/cloud/systemdSelfUpdate.acceptance.test.ts
	9	+ */
	10	+import * as NodeServices from "@effect/platform-node/NodeServices";
	11	+import { assert, it } from "@effect/vitest";
	12	+import * as Clock from "effect/Clock";
	13	+import * as Duration from "effect/Duration";
	14	+import * as Effect from "effect/Effect";
	15	+import * as FileSystem from "effect/FileSystem";
	16	+import * as Layer from "effect/Layer";
	17	+import * as Option from "effect/Option";
	18	+import * as Path from "effect/Path";
	19	+import * as Schema from "effect/Schema";
	20	+import { FetchHttpClient } from "effect/unstable/http";
	21	+import { describe } from "vite-plus/test";
	22	+
	23	+import * as ProcessRunner from "../processRunner.ts";
	24	+import { readPersistedServerRuntimeState } from "../serverRuntimeState.ts";
	25	+import { quoteSystemdValue } from "./bootService.ts";
	26	+import {
	27	+ SystemdSelfUpdatePlan,
	28	+ SystemdSelfUpdateReceipt,
	29	+ waitForSystemdSelfUpdateReadiness,
	30	+ writeSystemdSelfUpdatePlan,
	31	+} from "./systemdSelfUpdate.ts";
	32	+
	33	+const OLD_VERSION = "0.0.28";
	34	+const TARGET_VERSION = "0.0.29";
	35	+const decodeLaunch = Schema.decodeUnknownSync(
	36	+ Schema.fromJsonString(
	37	+ Schema.Struct({
	38	+ pid: Schema.Int,
	39	+ version: Schema.String,
	40	+ }),
	41	+ ),
	42	+);
	43	+
	44	+const fixtureServerSource = `
	45	+import * as fs from "node:fs";
	46	+import * as http from "node:http";
	47	+
	48	+const [runtimeStatePath, launchesPath, version, readiness] = process.argv.slice(2);
	49	+const descriptor = {
	50	+ environmentId: "systemd-self-update-acceptance",
	51	+ label: "Systemd self-update acceptance",
	52	+ platform: { os: "linux", arch: "x64" },
	53	+ serverVersion: version,
	54	+ capabilities: { repositoryIdentity: true },
	55	+};
	56	+const server = http.createServer((request, response) => {
	57	+ if (request.url === "/.well-known/t3/ready" && readiness === "ready") {
	58	+ response.writeHead(200, { "content-type": "application/json" });
	59	+ response.end(JSON.stringify(descriptor));
	60	+ return;
	61	+ }
	62	+ response.writeHead(503);
	63	+ response.end();
	64	+});
	65	+server.listen(0, "127.0.0.1", () => {
	66	+ const address = server.address();
	67	+ if (!address \|\| typeof address === "string") process.exit(1);
	68	+ fs.appendFileSync(launchesPath, JSON.stringify({ pid: process.pid, version }) + "\\n");
	69	+ fs.writeFileSync(runtimeStatePath, JSON.stringify({
	70	+ version: 1,
	71	+ pid: process.pid,
	72	+ port: address.port,
	73	+ origin: "http://127.0.0.1:" + address.port,
	74	+ startedAt: new Date().toISOString(),
	75	+ }) + "\\n");
	76	+});
	77	+`;
	78	+
	79	+const runAcceptanceScenario = Effect.fn("test.runSystemdSelfUpdateAcceptance")(function* (
	80	+ scenario: "healthy" \| "rollback",
	81	+) {
	82	+ const fs = yield* FileSystem.FileSystem;
	83	+ const path = yield* Path.Path;
	84	+ const runner = yield* ProcessRunner.ProcessRunner;
	85	+ const home = process.env.HOME;
	86	+ if (!home) {
	87	+ return yield* Effect.die(new Error("HOME is required for a systemd user-manager test."));
	88	+ }
	89	+
	90	+ const directory = yield* fs.makeTempDirectoryScoped({ prefix: `t3-systemd-${scenario}-` });
	91	+ const runtimeStatePath = path.join(directory, "server-runtime.json");
	92	+ const launchesPath = path.join(directory, "launches.jsonl");
	93	+ const receiptPath = path.join(directory, "receipt.json");
	94	+ const planPath = path.join(directory, "plan.json");
	95	+ const fixtureServerPath = path.join(directory, "fixture-server.mjs");
	96	+ yield* fs.writeFileString(fixtureServerPath, fixtureServerSource);
	97	+
	98	+ const suffix = `${String(process.pid)}-${String(yield* Clock.currentTimeMillis)}-${scenario}`;
	99	+ const serviceUnit = `t3code-self-update-acceptance-${suffix}.service`;
	100	+ const helperUnit = `t3code-self-update-acceptance-helper-${suffix}.service`;
	101	+ const userUnitDirectory = path.join(home, ".config", "systemd", "user");
	102	+ const unitPath = path.join(userUnitDirectory, serviceUnit);
	103	+ yield* fs.makeDirectory(userUnitDirectory, { recursive: true });
	104	+
	105	+ const renderUnit = (version: string, readiness: "ready" \| "blocked") =>
	106	+ [
	107	+ "[Unit]",
	108	+ "Description=T3 self-update acceptance fixture",
	109	+ "",
	110	+ "[Service]",
	111	+ "Type=simple",
	112	+ `ExecStart=${quoteSystemdValue(process.execPath)} ${quoteSystemdValue(fixtureServerPath)} ${quoteSystemdValue(runtimeStatePath)} ${quoteSystemdValue(launchesPath)} ${version} ${readiness}`,
	113	+ "Restart=no",
	114	+ "TimeoutStopSec=5",
	115	+ "",
	116	+ ].join("\n");
	117	+ const previousUnit = renderUnit(OLD_VERSION, "ready");
	118	+ const nextUnit = renderUnit(TARGET_VERSION, scenario === "healthy" ? "ready" : "blocked");
	119	+
	120	+ const systemctl = (args: ReadonlyArray<string>) =>
	121	+ runner.run({ command: "systemctl", args: ["--user", ...args] });
	122	+ const cleanup = Effect.gen(function* () {
	123	+ yield* systemctl(["stop", helperUnit, serviceUnit]).pipe(Effect.ignore);
	124	+ yield* systemctl(["reset-failed", helperUnit, serviceUnit]).pipe(Effect.ignore);
	125	+ yield* fs.remove(unitPath, { force: true }).pipe(Effect.ignore);
	126	+ yield* systemctl(["daemon-reload"]).pipe(Effect.ignore);
	127	+ });
	128	+ yield* Effect.addFinalizer(() => cleanup);
	129	+
	130	+ yield* fs.writeFileString(unitPath, previousUnit);
	131	+ const reload = yield* systemctl(["daemon-reload"]);
	132	+ assert.equal(Number(reload.code), 0);
	133	+ const start = yield* systemctl(["start", serviceUnit]);
	134	+ assert.equal(Number(start.code), 0);
	135	+ yield* waitForSystemdSelfUpdateReadiness(
	136	+ {
	137	+ expectedVersion: OLD_VERSION,
	138	+ previousPid: 0,
	139	+ runtimeStatePath,
	140	+ },
	141	+ { timeout: Duration.seconds(10), interval: Duration.millis(100) },
	142	+ );
	143	+ const initialState = yield* readPersistedServerRuntimeState(runtimeStatePath);
	144	+ if (Option.isNone(initialState)) {
	145	+ return yield* Effect.die(
	146	+ new Error("The disposable systemd service did not publish runtime state."),
	147	+ );
	148	+ }
	149	+
	150	+ const plan = new SystemdSelfUpdatePlan({
	151	+ version: 1,
	152	+ fromVersion: OLD_VERSION,
	153	+ targetVersion: TARGET_VERSION,
	154	+ currentPid: initialState.value.pid,
	155	+ serviceUnit,
	156	+ unitPath,
	157	+ previousUnit,
	158	+ nextUnit,
	159	+ runtimeStatePath,
	160	+ receiptPath,
	161	+ });
	162	+ yield* writeSystemdSelfUpdatePlan(planPath, plan);
	163	+
	164	+ const repositoryRoot = path.resolve(import.meta.dirname, "../../../..");
	165	+ const cliEntryPath = path.join(repositoryRoot, "apps", "server", "src", "bin.ts");
	166	+ const handoff = yield* runner.run({
	167	+ command: "systemd-run",
	168	+ args: [
	169	+ "--user",
	170	+ "--wait",
	171	+ "--collect",
	172	+ "--service-type=exec",
	173	+ `--unit=${helperUnit}`,
	174	+ process.execPath,
	175	+ "--experimental-strip-types",
	176	+ cliEntryPath,
	177	+ "service",
	178	+ "_apply-update",
	179	+ "--plan",
	180	+ planPath,
	181	+ ],
	182	+ timeout: Duration.seconds(90),
	183	+ });
	184	+ assert.equal(Number(handoff.code), 0, handoff.stderr);
	185	+
	186	+ const receipt = yield* fs
	187	+ .readFileString(receiptPath)
	188	+ .pipe(
	189	+ Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(SystemdSelfUpdateReceipt))),
	190	+ );
	191	+ assert.equal(receipt.phase, scenario === "healthy" ? "healthy" : "rolled-back");
	192	+ assert.equal(
	193	+ yield* fs.readFileString(unitPath),
	194	+ scenario === "healthy" ? nextUnit : previousUnit,
	195	+ );
	196	+
	197	+ yield* waitForSystemdSelfUpdateReadiness(
	198	+ {
	199	+ expectedVersion: scenario === "healthy" ? TARGET_VERSION : OLD_VERSION,
	200	+ previousPid: initialState.value.pid,
	201	+ runtimeStatePath,
	202	+ },
	203	+ { timeout: Duration.seconds(10), interval: Duration.millis(100) },
	204	+ );
	205	+ const launches = (yield* fs.readFileString(launchesPath))
	206	+ .trim()
	207	+ .split("\n")
	208	+ .map((line) => decodeLaunch(line));
	209	+ assert.deepEqual(
	210	+ launches.map(({ version }) => version),
	211	+ scenario === "healthy"
	212	+ ? [OLD_VERSION, TARGET_VERSION]
	213	+ : [OLD_VERSION, TARGET_VERSION, OLD_VERSION],
	214	+ );
	215	+});
	216	+
	217	+describe.runIf(process.env.T3_SYSTEMD_SELF_UPDATE_ACCEPTANCE === "1")(
	218	+ "systemd self-update acceptance",
	219	+ () => {
	220	+ const layer = ProcessRunner.layer.pipe(
	221	+ Layer.provideMerge(NodeServices.layer),
	222	+ Layer.merge(FetchHttpClient.layer),
	223	+ );
	224	+
	225	+ it.live(
	226	+ "keeps the helper alive while replacing the host service",
	227	+ () => runAcceptanceScenario("healthy").pipe(Effect.provide(layer)),
	228	+ 30_000,
	229	+ );
	230	+
	231	+ it.live(
	232	+ "rolls back a server that binds HTTP without becoming runtime-ready",
	233	+ () => runAcceptanceScenario("rollback").pipe(Effect.provide(layer)),
	234	+ 120_000,
	235	+ );
	236	+ },
	237	+);

apps/server/src/cloud/systemdSelfUpdate.test.ts+56−4

@@ -6,14 +6,17 @@ import * as FileSystem from "effect/FileSystem"; · 5 unmodified lines
6	6	import * as Layer from "effect/Layer";
7	7	import * as Path from "effect/Path";
8	8	import * as Schema from "effect/Schema";
	9	+import { HttpClient, HttpClientResponse } from "effect/unstable/http";
9	10	import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";
10	11
11	12	import * as ProcessRunner from "../processRunner.ts";
	13	+import { PersistedServerRuntimeState } from "../serverRuntimeState.ts";
12	14	import {
13	15	SystemdSelfUpdateActivationError,
14	16	SystemdSelfUpdatePlan,
15	17	SystemdSelfUpdateReceipt,
16	18	activateSystemdSelfUpdatePlan,
	19	+ waitForSystemdSelfUpdateReadiness,
17	20	} from "./systemdSelfUpdate.ts";
18	21
19	22	interface RecordedCommand {
@@ -59,6 +62,7 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { · 39 unmodified lines
59	62	fromVersion: "0.0.28",
60	63	targetVersion: "0.0.29",
61	64	currentPid: 123,
	65	+ serviceUnit: "t3code-self-update-test.service",
62	66	unitPath,
63	67	previousUnit,
64	68	nextUnit: "new unit\n",
@@ -95,7 +99,7 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { · 30 unmodified lines
95	99	commands.map(({ args }) => args),
96	100	[
97	101	["--user", "daemon-reload"],
98		− ["--user", "restart", "t3code.service"],
	102	+ ["--user", "restart", "t3code-self-update-test.service"],
99	103	],
100	104	);
101	105	assert.equal((yield* readReceipt(fixture.receiptPath)).phase, "healthy");
@@ -132,10 +136,10 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { · 30 unmodified lines
132	136	commands.map(({ args }) => args),
133	137	[
134	138	["--user", "daemon-reload"],
135		− ["--user", "restart", "t3code.service"],
	139	+ ["--user", "restart", "t3code-self-update-test.service"],
136	140	["--user", "daemon-reload"],
137		− ["--user", "reset-failed", "t3code.service"],
138		− ["--user", "restart", "t3code.service"],
	141	+ ["--user", "reset-failed", "t3code-self-update-test.service"],
	142	+ ["--user", "restart", "t3code-self-update-test.service"],
139	143	],
140	144	);
141	145	const receipt = yield* readReceipt(fixture.receiptPath);
@@ -164,4 +168,52 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { · 22 unmodified lines
164	168	assert.equal((yield* readReceipt(fixture.receiptPath)).phase, "recovery-failed");
165	169	}),
166	170	);
	171	+
	172	+ it.effect("probes the runtime-readiness endpoint on the replacement process", () =>
	173	+ Effect.gen(function* () {
	174	+ const fs = yield* FileSystem.FileSystem;
	175	+ const path = yield* Path.Path;
	176	+ const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-systemd-ready-test-" });
	177	+ const runtimeStatePath = path.join(directory, "server-runtime.json");
	178	+ const encodeRuntimeState = Schema.encodeEffect(
	179	+ Schema.fromJsonString(PersistedServerRuntimeState),
	180	+ );
	181	+ const encodedRuntimeState = yield* encodeRuntimeState({
	182	+ version: 1,
	183	+ pid: 456,
	184	+ port: 3210,
	185	+ origin: "http://127.0.0.1:3210",
	186	+ startedAt: "2026-01-01T00:00:00.000Z",
	187	+ });
	188	+ yield* fs.writeFileString(runtimeStatePath, `${encodedRuntimeState}\n`);
	189	+
	190	+ const requests: Array<string> = [];
	191	+ const httpClientLayer = Layer.succeed(
	192	+ HttpClient.HttpClient,
	193	+ HttpClient.make((request) => {
	194	+ requests.push(request.url);
	195	+ return Effect.succeed(
	196	+ HttpClientResponse.fromWeb(
	197	+ request,
	198	+ Response.json({
	199	+ environmentId: "environment-test",
	200	+ label: "Test environment",
	201	+ platform: { os: "linux", arch: "x64" },
	202	+ serverVersion: "0.0.29",
	203	+ capabilities: { repositoryIdentity: true },
	204	+ }),
	205	+ ),
	206	+ );
	207	+ }),
	208	+ );
	209	+
	210	+ yield* waitForSystemdSelfUpdateReadiness({
	211	+ expectedVersion: "0.0.29",
	212	+ previousPid: 123,
	213	+ runtimeStatePath,
	214	+ }).pipe(Effect.provide(httpClientLayer));
	215	+
	216	+ assert.deepEqual(requests, ["http://127.0.0.1:3210/.well-known/t3/ready"]);
	217	+ }),
	218	+ );
167	219	});

apps/server/src/cloud/systemdSelfUpdate.ts+8−7

@@ -12,7 +12,6 @@ import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstab · 11 unmodified lines
12	12	import { writeFileStringAtomically } from "../atomicWrite.ts";
13	13	import * as ProcessRunner from "../processRunner.ts";
14	14	import { readPersistedServerRuntimeState } from "../serverRuntimeState.ts";
15		−import { BOOT_SERVICE_UNIT_FILE } from "./bootService.ts";
16	15
17	16	export const SYSTEMD_SELF_UPDATE_UNIT = "t3code-self-update.service";
18	17	export const SYSTEMD_SELF_UPDATE_DIRECTORY = "self-update";
@@ -23,7 +22,7 @@ const DEFAULT_RESTART_DELAY = Duration.seconds(2); · 4 unmodified lines
23	22	const DEFAULT_READINESS_TIMEOUT = Duration.seconds(60);
24	23	const DEFAULT_READINESS_INTERVAL = Duration.millis(500);
25	24	const READINESS_REQUEST_TIMEOUT = Duration.seconds(2);
26		−const WELL_KNOWN_ENVIRONMENT_PATH = "/.well-known/t3/environment";
	25	+const WELL_KNOWN_READY_PATH = "/.well-known/t3/ready";
27	26
28	27	export class SystemdSelfUpdatePlan extends Schema.Class<SystemdSelfUpdatePlan>(
29	28	"SystemdSelfUpdatePlan",
@@ -32,6 +31,7 @@ export class SystemdSelfUpdatePlan extends Schema.Class<SystemdSelfUpdatePlan>( · 2 unmodified lines
32	31	fromVersion: Schema.String,
33	32	targetVersion: Schema.String,
34	33	currentPid: Schema.Int,
	34	+ serviceUnit: Schema.String,
35	35	unitPath: Schema.String,
36	36	previousUnit: Schema.String,
37	37	nextUnit: Schema.String,
@@ -185,7 +185,8 @@ const installUnit = Effect.fn("systemdSelfUpdate.installUnit")(function* ( · 147 unmodified lines
185	185	yield* runSystemctl(["--user", "daemon-reload"], "daemon-reload");
186	186	});
187	187
188		−const restartBootService = runSystemctl(["--user", "restart", BOOT_SERVICE_UNIT_FILE], "restart");
	188	+const restartBootService = (serviceUnit: string) =>
	189	+ runSystemctl(["--user", "restart", serviceUnit], "restart");
189	190
190	191	export interface SystemdSelfUpdateReadinessInput {
191	192	readonly expectedVersion: string;
@@ -227,7 +228,7 @@ export const waitForSystemdSelfUpdateReadiness = Effect.fn("systemdSelfUpdate.wa · 35 unmodified lines
227	228	}
228	229
229	230	const request = HttpClientRequest.get(
230		− new URL(WELL_KNOWN_ENVIRONMENT_PATH, state.value.origin).toString(),
	231	+ new URL(WELL_KNOWN_READY_PATH, state.value.origin).toString(),
231	232	);
232	233	const descriptor = yield* client.execute(request).pipe(
233	234	Effect.timeout(READINESS_REQUEST_TIMEOUT),
@@ -277,7 +278,7 @@ export const activateSystemdSelfUpdatePlan = Effect.fn("systemdSelfUpdate.activa · 43 unmodified lines
277	278	yield* writeReceipt(plan, "activating").pipe(Effect.ignore);
278	279	yield* Effect.sleep(options.restartDelay ?? DEFAULT_RESTART_DELAY);
279	280	yield* installUnit(plan, plan.nextUnit);
280		− yield* restartBootService;
	281	+ yield* restartBootService(plan.serviceUnit);
281	282	yield* waitForReadiness({
282	283	expectedVersion: plan.targetVersion,
283	284	previousPid: plan.currentPid,
@@ -298,8 +299,8 @@ export const activateSystemdSelfUpdatePlan = Effect.fn("systemdSelfUpdate.activa · 14 unmodified lines
298	299	);
299	300	const recovery = Effect.gen(function* () {
300	301	yield* installUnit(plan, plan.previousUnit);
301		− yield* runSystemctl(["--user", "reset-failed", BOOT_SERVICE_UNIT_FILE], "reset-failed");
302		− yield* restartBootService;
	302	+ yield* runSystemctl(["--user", "reset-failed", plan.serviceUnit], "reset-failed");
	303	+ yield* restartBootService(plan.serviceUnit);
303	304	yield* waitForReadiness({
304	305	expectedVersion: plan.fromVersion,
305	306	previousPid: plan.currentPid,

apps/server/src/http.ts+18−7

@@ -40,6 +40,7 @@ import { · 39 unmodified lines
40	40	} from "./auth/http.ts";
41	41	import * as ServerEnvironment from "./environment/ServerEnvironment.ts";
42	42	import { browserApiCorsAllowedHeaders, browserApiCorsAllowedMethods } from "./httpCors.ts";
	43	+import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts";
43	44
44	45	const OTLP_TRACES_PROXY_PATH = "/api/observability/v1/traces";
45	46	const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]);
@@ -177,13 +178,23 @@ export const serverEnvironmentHttpApiLayer = HttpApiBuilder.group( · 131 unmodified lines
177	178	"metadata",
178	179	Effect.fnUntraced(function* (handlers) {
179	180	const serverEnvironment = yield* ServerEnvironment.ServerEnvironment;
180		− return handlers.handle(
181		− "descriptor",
182		− Effect.fn("environment.metadata.descriptor")(function* (args) {
183		− yield* annotateEnvironmentRequest(args.endpoint.name);
184		− return yield* serverEnvironment.getDescriptor;
185		− }, traceRelayRequest),
186		− );
	181	+ const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup;
	182	+ return handlers
	183	+ .handle(
	184	+ "descriptor",
	185	+ Effect.fn("environment.metadata.descriptor")(function* (args) {
	186	+ yield* annotateEnvironmentRequest(args.endpoint.name);
	187	+ return yield* serverEnvironment.getDescriptor;
	188	+ }, traceRelayRequest),
	189	+ )
	190	+ .handle(
	191	+ "ready",
	192	+ Effect.fn("environment.metadata.ready")(function* (args) {
	193	+ yield* annotateEnvironmentRequest(args.endpoint.name);
	194	+ yield* startup.awaitReady.pipe(Effect.orDie);
	195	+ return yield* serverEnvironment.getDescriptor;
	196	+ }, traceRelayRequest),
	197	+ );
187	198	}),
188	199	);
189	200

apps/server/src/persistence/Migrations.test.ts+51−1

@@ -3,7 +3,12 @@ import * as Effect from "effect/Effect"; · 2 unmodified lines
3	3	import * as Layer from "effect/Layer";
4	4	import * as SqlClient from "effect/unstable/sql/SqlClient";
5	5
6		−import { MigrationIdentityMismatchError, validateMigrationIdentities } from "./Migrations.ts";
	6	+import {
	7	+ AutomaticUpdateMigrationFrontierError,
	8	+ MigrationIdentityMismatchError,
	9	+ validateAutomaticUpdateMigrationFrontier,
	10	+ validateMigrationIdentities,
	11	+} from "./Migrations.ts";
7	12	import * as NodeSqliteClient from "./NodeSqliteClient.ts";
8	13
9	14	const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory()));
@@ -50,4 +55,49 @@ layer("migration identity validation", (it) => { · 40 unmodified lines
50	55	yield* validateMigrationIdentities();
51	56	}),
52	57	);
	58	+
	59	+ it.effect("rejects automatic downgrade when the database has a newer migration", () =>
	60	+ Effect.gen(function* () {
	61	+ const sql = yield* SqlClient.SqlClient;
	62	+ yield* createTrackingTable();
	63	+ yield* sql`
	64	+ INSERT INTO effect_sql_migrations (migration_id, name)
	65	+ VALUES (999, 'FutureMigration')
	66	+ `;
	67	+
	68	+ const error = yield* validateAutomaticUpdateMigrationFrontier().pipe(Effect.flip);
	69	+ assert.instanceOf(error, AutomaticUpdateMigrationFrontierError);
	70	+ assert.equal(error.databaseMigrationId, 999);
	71	+ assert.equal(error.targetMigrationId, 35);
	72	+ }),
	73	+ );
	74	+
	75	+ it.effect("rejects automatic updates that would run migrations", () =>
	76	+ Effect.gen(function* () {
	77	+ const sql = yield* SqlClient.SqlClient;
	78	+ yield* createTrackingTable();
	79	+ yield* sql`
	80	+ INSERT INTO effect_sql_migrations (migration_id, name)
	81	+ VALUES (34, 'ProjectionThreadsSnoozed')
	82	+ `;
	83	+
	84	+ const error = yield* validateAutomaticUpdateMigrationFrontier().pipe(Effect.flip);
	85	+ assert.instanceOf(error, AutomaticUpdateMigrationFrontierError);
	86	+ assert.equal(error.databaseMigrationId, 34);
	87	+ assert.equal(error.targetMigrationId, 35);
	88	+ }),
	89	+ );
	90	+
	91	+ it.effect("allows automatic updates on the same migration frontier", () =>
	92	+ Effect.gen(function* () {
	93	+ const sql = yield* SqlClient.SqlClient;
	94	+ yield* createTrackingTable();
	95	+ yield* sql`
	96	+ INSERT INTO effect_sql_migrations (migration_id, name)
	97	+ VALUES (35, 'ProjectionThreadTitleRegeneration')
	98	+ `;
	99	+
	100	+ yield* validateAutomaticUpdateMigrationFrontier();
	101	+ }),
	102	+ );
53	103	});

apps/server/src/persistence/Migrations.ts+58−18

@@ -127,33 +127,53 @@ export class MigrationIdentityMismatchError extends Schema.TaggedErrorClass<Migr · 126 unmodified lines
127	127	}
128	128	}
129	129
	130	+export class AutomaticUpdateMigrationFrontierError extends Schema.TaggedErrorClass<AutomaticUpdateMigrationFrontierError>()(
	131	+ "AutomaticUpdateMigrationFrontierError",
	132	+ {
	133	+ databaseMigrationId: Schema.Int,
	134	+ targetMigrationId: Schema.Int,
	135	+ },
	136	+) {
	137	+ override get message(): string {
	138	+ return this.databaseMigrationId > this.targetMigrationId
	139	+ ? `This database is newer than the requested t3 build (migration ${String(this.databaseMigrationId)} vs ${String(this.targetMigrationId)}).`
	140	+ : `The requested t3 build would migrate this database (migration ${String(this.databaseMigrationId)} to ${String(this.targetMigrationId)}), which is not safe during an automatic update.`;
	141	+ }
	142	+}
	143	+
	144	+const readPersistedMigrationIdentities = Effect.fn("readPersistedMigrationIdentities")(
	145	+ function* () {
	146	+ const sql = yield* SqlClient.SqlClient;
	147	+ const tables = yield* sql<{ readonly name: string }>`
	148	+ SELECT name
	149	+ FROM sqlite_master
	150	+ WHERE type = 'table' AND name = 'effect_sql_migrations'
	151	+ `;
	152	+ if (tables.length === 0) {
	153	+ return [];
	154	+ }
	155	+
	156	+ return yield* sql<{
	157	+ readonly migration_id: number;
	158	+ readonly name: string;
	159	+ }>`
	160	+ SELECT migration_id, name
	161	+ FROM effect_sql_migrations
	162	+ ORDER BY migration_id
	163	+ `;
	164	+ },
	165	+);
	166	+
130	167	/**
131	168	* Refuse to run a build whose migration history disagrees with the database.
132	169	* Unknown newer IDs are intentionally allowed so a compatible older build can
133	170	* inspect a database created by a newer release without rewriting its history.
134	171	*/
135	172	export const validateMigrationIdentities = Effect.fn("validateMigrationIdentities")(function* () {
136		− const sql = yield* SqlClient.SqlClient;
137		− const tables = yield* sql<{ readonly name: string }>`
138		− SELECT name
139		− FROM sqlite_master
140		− WHERE type = 'table' AND name = 'effect_sql_migrations'
141		− `;
142		− if (tables.length === 0) {
143		− return;
144		− }
145		−
146	173	const expectedNames = new Map<number, string>(
147	174	migrationEntries.map(([migrationId, name]) => [migrationId, name]),
148	175	);
149		− const persisted = yield* sql<{
150		− readonly migration_id: number;
151		− readonly name: string;
152		− }>`
153		− SELECT migration_id, name
154		− FROM effect_sql_migrations
155		− ORDER BY migration_id
156		− `;
	176	+ const persisted = yield* readPersistedMigrationIdentities();
157	177
158	178	for (const migration of persisted) {
159	179	const expectedName = expectedNames.get(migration.migration_id);
@@ -167,6 +187,26 @@ export const validateMigrationIdentities = Effect.fn("validateMigrationIdentitie · 7 unmodified lines
167	187	}
168	188	});
169	189
	190	+/**
	191	+ * Automatic activation is safe only when the target and database are already
	192	+ * on the same schema. A binary rollback cannot undo a migration, and an older
	193	+ * target cannot safely interpret migration IDs that it does not know about.
	194	+ */
	195	+export const validateAutomaticUpdateMigrationFrontier = Effect.fn(
	196	+ "validateAutomaticUpdateMigrationFrontier",
	197	+)(function* () {
	198	+ yield* validateMigrationIdentities();
	199	+ const persisted = yield* readPersistedMigrationIdentities();
	200	+ const databaseMigrationId = persisted.at(-1)?.migration_id ?? 0;
	201	+ const targetMigrationId = migrationEntries.at(-1)?.[0] ?? 0;
	202	+ if (databaseMigrationId !== targetMigrationId) {
	203	+ return yield* new AutomaticUpdateMigrationFrontierError({
	204	+ databaseMigrationId,
	205	+ targetMigrationId,
	206	+ });
	207	+ }
	208	+});
	209	+
170	210	export interface RunMigrationsOptions {
171	211	readonly toMigrationInclusive?: number \| undefined;
172	212	}

apps/server/src/server.test.ts+25−0

@@ -782,6 +782,7 @@ const buildAppUnderTest = (options?: { · 781 unmodified lines
782	782	Layer.provide(
783	783	Layer.mock(ServerRuntimeStartup.ServerRuntimeStartup)({
784	784	awaitCommandReady: Effect.void,
	785	+ awaitReady: Effect.void,
785	786	markHttpListening: Effect.void,
786	787	enqueueCommand: (effect) => effect,
787	788	...options?.layers?.serverRuntimeStartup,
@@ -1361,6 +1362,30 @@ it.layer(NodeServices.layer)("server router seam", (it) => { · 573 unmodified lines
1361	1362	}).pipe(Effect.provide(NodeHttpServer.layerTest)),
1362	1363	);
1363	1364
	1365	+ it.effect("does not report update readiness before the runtime publishes ready", () =>
	1366	+ Effect.gen(function* () {
	1367	+ const runtimeReady = yield* Deferred.make<void>();
	1368	+ yield* buildAppUnderTest({
	1369	+ layers: {
	1370	+ serverRuntimeStartup: {
	1371	+ awaitReady: Deferred.await(runtimeReady),
	1372	+ },
	1373	+ },
	1374	+ });
	1375	+
	1376	+ const url = yield* getHttpServerUrl("/.well-known/t3/ready");
	1377	+ const responseFiber = yield* fetchEffect(url).pipe(Effect.forkScoped);
	1378	+ yield* Effect.yieldNow;
	1379	+ assert.isUndefined(responseFiber.pollUnsafe());
	1380	+
	1381	+ yield* Deferred.succeed(runtimeReady, undefined);
	1382	+ const response = yield* Fiber.join(responseFiber);
	1383	+ const body = yield* responseJsonEffect<typeof testEnvironmentDescriptor>(response);
	1384	+ assert.equal(response.status, 200);
	1385	+ assert.deepEqual(body, testEnvironmentDescriptor);
	1386	+ }).pipe(Effect.provide(NodeHttpServer.layerTest)),
	1387	+ );
	1388	+
1364	1389	it.effect("compresses large JSON responses through the composed routes", () =>
1365	1390	Effect.gen(function* () {
1366	1391	const descriptor = {

apps/server/src/serverRuntimeStartup.ts+5−0

@@ -59,6 +59,7 @@ export class ServerRuntimeStartup extends Context.Service< · 58 unmodified lines
59	59	ServerRuntimeStartup,
60	60	{
61	61	readonly awaitCommandReady: Effect.Effect<void, ServerRuntimeStartupError>;
	62	+ readonly awaitReady: Effect.Effect<void, ServerRuntimeStartupError>;
62	63	readonly markHttpListening: Effect.Effect<void>;
63	64	readonly enqueueCommand: <A, E>(
64	65	effect: Effect.Effect<A, E>,
@@ -300,6 +301,7 @@ export const make = Effect.gen(function* () { · 235 unmodified lines
300	301
301	302	const commandGate = yield* makeCommandGate;
302	303	const httpListening = yield* Deferred.make<void>();
	304	+ const runtimeReady = yield* Deferred.make<void, ServerRuntimeStartupError>();
303	305	const reactorScope = yield* Scope.make("sequential");
304	306
305	307	yield* Effect.addFinalizer(() => Scope.close(reactorScope, Exit.void));
@@ -425,6 +427,7 @@ export const make = Effect.gen(function* () { · 119 unmodified lines
425	427	});
426	428	yield* Effect.logError("server runtime startup failed", { cause: startupExit.cause });
427	429	yield* commandGate.failCommandReady(error);
	430	+ yield* Deferred.fail(runtimeReady, error).pipe(Effect.orDie);
428	431	return;
429	432	}
430	433
@@ -444,6 +447,7 @@ export const make = Effect.gen(function* () { · 13 unmodified lines
444	447	},
445	448	}),
446	449	);
	450	+ yield* Deferred.succeed(runtimeReady, undefined).pipe(Effect.orDie);
447	451
448	452	yield* Effect.logDebug("startup phase: recording startup heartbeat");
449	453	yield* launchStartupHeartbeat;
@@ -470,6 +474,7 @@ export const make = Effect.gen(function* () { · 20 unmodified lines
470	474
471	475	return {
472	476	awaitCommandReady: commandGate.awaitCommandReady,
	477	+ awaitReady: Deferred.await(runtimeReady),
473	478	markHttpListening: Deferred.succeed(httpListening, undefined),
474	479	enqueueCommand: commandGate.enqueueCommand,
475	480	} satisfies ServerRuntimeStartup["Service"];

docs/internals/server-updates.md+9−4

@@ -90,9 +90,12 @@ successfully. Boot-service setup and self-update share the same process-wide ins · 89 unmodified lines
90	90	they cannot mutate a pinned runtime concurrently.
91	91
92	92	Before any restart, the current Node executable runs the replacement with `--version`, then asks
93		−the target artifact to validate its migration identities against the live database in read-only
94		−mode. A failed install, version preflight, or compatibility check leaves the current server running.
95		−Targets old enough to lack the compatibility command are not eligible for remote activation.
	93	+the target artifact to validate the live database in read-only mode. Automatic activation requires
	94	+the target and database to be on the exact same migration frontier. This rejects both downgrades and
	95	+updates that would migrate the database because a binary rollback cannot undo a schema change. Use
	96	+the manual host service update path for those releases. A failed install, version preflight, or
	97	+compatibility check leaves the current server running. Targets old enough to lack the compatibility
	98	+command are not eligible for remote activation.
96	99
97	100	Candidate cleanup is narrower than "any failed preflight". The candidate runtime is removed only when
98	101	the preflight process actually completes and reports a bad exit code or the wrong version: that is
@@ -118,7 +121,9 @@ For `boot-service`, the request process writes a typed plan but leaves its live · 19 unmodified lines
118	121	starts a fixed-name `t3code-self-update.service` transient unit with `systemd-run --user`; the
119	122	transient unit has a separate cgroup and acts as the cross-process update lock. After a short grace
120	123	period it atomically installs the candidate unit, reloads systemd, performs a blocking restart, and
121		−probes the public environment descriptor for the exact target version. If activation or readiness
	124	+probes the public runtime-readiness endpoint for the exact target version. The endpoint does not
	125	+respond until the server's command runtime is usable and its lifecycle `ready` event has been
	126	+published, so an HTTP listener alone cannot make an activation healthy. If activation or readiness
122	127	fails, the helper restores the previous unit, clears systemd's failed state, restarts it, and proves
123	128	the previous version is ready. The plan and latest receipt live under the environment state
124	129	directory's `self-update` folder for diagnosis.

docs/operations/release.md+17−0

@@ -190,6 +190,23 @@ connect the new client to a server on the previous version and verify that the u · 189 unmodified lines
190	190	reconnects to the matching server. Test one automatic path and the manual or desktop-managed
191	191	guidance when those environments are available.
192	192
	193	+The automatic boot-service path also requires a real systemd user-manager acceptance run. Use a
	194	+disposable T3 home and uniquely named host and helper units, never the developer's daily
	195	+`t3code.service`. Exercise both outcomes:
	196	+
	197	+1. A same-migration-frontier candidate becomes runtime-ready, the helper survives the host-unit
	198	+ restart, and the receipt reaches `healthy` with a new runtime PID and the exact target version.
	199	+2. A candidate that binds HTTP but never becomes runtime-ready times out, the previous unit is
	200	+ restored, and the receipt reaches `rolled-back` with the old version command-ready again.
	201	+
	202	+Treat the systemd run as a release blocker when the update lifecycle changes. CI jobs without a
	203	+user manager may skip it, but a Linux release host must record the run before publishing.
	204	+
	205	+```bash
	206	+T3_SYSTEMD_SELF_UPDATE_ACCEPTANCE=1 vp test run \
	207	+ apps/server/src/cloud/systemdSelfUpdate.acceptance.test.ts
	208	+```
	209	+
193	210	## Desktop auto-update notes
194	211
195	212	- Updater runtime: `apps/desktop/src/updates/DesktopUpdates.ts`.

packages/contracts/src/environmentHttp.ts+11−5

@@ -371,11 +371,17 @@ export const AuthOtherClientSessionsRevokeResult = Schema.Struct({ · 370 unmodified lines
371	371	});
372	372	export type AuthOtherClientSessionsRevokeResult = typeof AuthOtherClientSessionsRevokeResult.Type;
373	373
374		−export class EnvironmentMetadataHttpApi extends HttpApiGroup.make("metadata").add(
375		− HttpApiEndpoint.get("descriptor", "/.well-known/t3/environment", {
376		− success: ExecutionEnvironmentDescriptor,
377		− }),
378		−) {}
	374	+export class EnvironmentMetadataHttpApi extends HttpApiGroup.make("metadata")
	375	+ .add(
	376	+ HttpApiEndpoint.get("descriptor", "/.well-known/t3/environment", {
	377	+ success: ExecutionEnvironmentDescriptor,
	378	+ }),
	379	+ )
	380	+ .add(
	381	+ HttpApiEndpoint.get("ready", "/.well-known/t3/ready", {
	382	+ success: ExecutionEnvironmentDescriptor,
	383	+ }),
	384	+ ) {}
379	385
380	386	export class EnvironmentAuthHttpApi extends HttpApiGroup.make("auth")
381	387	.add(

fix(server): validate update rollback boundaries

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code