History

fix(server): validate update rollback boundaries

Theo Browne committed 3 months ago

Commit cd25a3d1 parent c07288814 files · +499 −48Browse files

Unified · Split

14 files
  1. apps/server/src/cli/service.ts +2 −2
  2. apps/server/src/cloud/selfUpdate.test.ts +1 −0
  3. apps/server/src/cloud/selfUpdate.ts +1 −0
  4. apps/server/src/cloud/systemdSelfUpdate.acceptance.test.ts +237 −0
  5. apps/server/src/cloud/systemdSelfUpdate.test.ts +56 −4
  6. apps/server/src/cloud/systemdSelfUpdate.ts +8 −7
  7. apps/server/src/http.ts +18 −7
  8. apps/server/src/persistence/Migrations.test.ts +51 −1
  9. apps/server/src/persistence/Migrations.ts +58 −18
  10. apps/server/src/server.test.ts +25 −0
  11. apps/server/src/serverRuntimeStartup.ts +5 −0
  12. docs/internals/server-updates.md +9 −4
  13. docs/operations/release.md +17 −0
  14. packages/contracts/src/environmentHttp.ts +11 −5
apps/server/src/cli/service.ts+2−2
@@ -9,7 +9,7 @@ import packageJson from "../../package.json" with { type: "json" }; · 8 unmodified lines
9import * as BootService from "../cloud/bootService.ts";9import * as BootService from "../cloud/bootService.ts";
10import { applySystemdSelfUpdatePlan } from "../cloud/systemdSelfUpdate.ts";10import { applySystemdSelfUpdatePlan } from "../cloud/systemdSelfUpdate.ts";
11import type * as ServerConfig from "../config.ts";11import type * as ServerConfig from "../config.ts";
12import { validateMigrationIdentities } from "../persistence/Migrations.ts";12import { validateAutomaticUpdateMigrationFrontier } from "../persistence/Migrations.ts";
13import * as ProcessRunner from "../processRunner.ts";13import * as ProcessRunner from "../processRunner.ts";
14import { projectLocationFlags, resolveCliAuthConfig } from "./config.ts";14import { projectLocationFlags, resolveCliAuthConfig } from "./config.ts";
15 15
@@ -169,7 +169,7 @@ const serviceValidateUpdateCommand = Command.make("_validate-update", { · 153 unmodified lines
169 const NodeSqliteClient = yield* Effect.promise(169 const NodeSqliteClient = yield* Effect.promise(
170 () => import("../persistence/NodeSqliteClient.ts"),170 () => import("../persistence/NodeSqliteClient.ts"),
171 );171 );
172 yield* validateMigrationIdentities().pipe(172 yield* validateAutomaticUpdateMigrationFrontier().pipe(
173 Effect.scoped,173 Effect.scoped,
174 Effect.provide(NodeSqliteClient.layer({ filename: database, readonly: true })),174 Effect.provide(NodeSqliteClient.layer({ filename: database, readonly: true })),
175 );175 );
apps/server/src/cloud/selfUpdate.test.ts+1−0
@@ -535,6 +535,7 @@ it.layer(NodeServices.layer)("ServerSelfUpdate.update", (it) => { · 534 unmodified lines
535 Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(SystemdSelfUpdatePlan))),535 Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(SystemdSelfUpdatePlan))),
536 );536 );
537 assert.equal(plan.previousUnit, previousUnit);537 assert.equal(plan.previousUnit, previousUnit);
538 assert.equal(plan.serviceUnit, BOOT_SERVICE_UNIT_FILE);
538 assert.include(plan.nextUnit, `ExecStart=${NODE_PATH} ${pinnedEntry} serve`);539 assert.include(plan.nextUnit, `ExecStart=${NODE_PATH} ${pinnedEntry} serve`);
539 assert.equal(540 assert.equal(
540 plan.runtimeStatePath,541 plan.runtimeStatePath,
apps/server/src/cloud/selfUpdate.ts+1−0
@@ -394,6 +394,7 @@ export const make = Effect.fn("cloud.server_self_update.make")(function* (option · 393 unmodified lines
394 fromVersion: packageJson.version,394 fromVersion: packageJson.version,
395 targetVersion,395 targetVersion,
396 currentPid: process.pid,396 currentPid: process.pid,
397 serviceUnit: BOOT_SERVICE_UNIT_FILE,
397 unitPath,398 unitPath,
398 previousUnit,399 previousUnit,
399 nextUnit,400 nextUnit,
apps/server/src/cloud/systemdSelfUpdate.acceptance.test.ts+237−0
@@ -0,0 +1,237 @@
1// @effect-diagnostics nodeBuiltinImport:off
2/**
3 * Opt-in acceptance coverage for the real systemd handoff and rollback path.
4 * Every unit and file is uniquely named; this must never target t3code.service.
5 *
6 * Run on Linux with a user manager:
7 * T3_SYSTEMD_SELF_UPDATE_ACCEPTANCE=1 vp test run \
8 * apps/server/src/cloud/systemdSelfUpdate.acceptance.test.ts
9 */
10import * as NodeServices from "@effect/platform-node/NodeServices";
11import { assert, it } from "@effect/vitest";
12import * as Clock from "effect/Clock";
13import * as Duration from "effect/Duration";
14import * as Effect from "effect/Effect";
15import * as FileSystem from "effect/FileSystem";
16import * as Layer from "effect/Layer";
17import * as Option from "effect/Option";
18import * as Path from "effect/Path";
19import * as Schema from "effect/Schema";
20import { FetchHttpClient } from "effect/unstable/http";
21import { describe } from "vite-plus/test";
22
23import * as ProcessRunner from "../processRunner.ts";
24import { readPersistedServerRuntimeState } from "../serverRuntimeState.ts";
25import { quoteSystemdValue } from "./bootService.ts";
26import {
27 SystemdSelfUpdatePlan,
28 SystemdSelfUpdateReceipt,
29 waitForSystemdSelfUpdateReadiness,
30 writeSystemdSelfUpdatePlan,
31} from "./systemdSelfUpdate.ts";
32
33const OLD_VERSION = "0.0.28";
34const TARGET_VERSION = "0.0.29";
35const decodeLaunch = Schema.decodeUnknownSync(
36 Schema.fromJsonString(
37 Schema.Struct({
38 pid: Schema.Int,
39 version: Schema.String,
40 }),
41 ),
42);
43
44const fixtureServerSource = `
45import * as fs from "node:fs";
46import * as http from "node:http";
47
48const [runtimeStatePath, launchesPath, version, readiness] = process.argv.slice(2);
49const descriptor = {
50 environmentId: "systemd-self-update-acceptance",
51 label: "Systemd self-update acceptance",
52 platform: { os: "linux", arch: "x64" },
53 serverVersion: version,
54 capabilities: { repositoryIdentity: true },
55};
56const server = http.createServer((request, response) => {
57 if (request.url === "/.well-known/t3/ready" && readiness === "ready") {
58 response.writeHead(200, { "content-type": "application/json" });
59 response.end(JSON.stringify(descriptor));
60 return;
61 }
62 response.writeHead(503);
63 response.end();
64});
65server.listen(0, "127.0.0.1", () => {
66 const address = server.address();
67 if (!address || typeof address === "string") process.exit(1);
68 fs.appendFileSync(launchesPath, JSON.stringify({ pid: process.pid, version }) + "\\n");
69 fs.writeFileSync(runtimeStatePath, JSON.stringify({
70 version: 1,
71 pid: process.pid,
72 port: address.port,
73 origin: "http://127.0.0.1:" + address.port,
74 startedAt: new Date().toISOString(),
75 }) + "\\n");
76});
77`;
78
79const runAcceptanceScenario = Effect.fn("test.runSystemdSelfUpdateAcceptance")(function* (
80 scenario: "healthy" | "rollback",
81) {
82 const fs = yield* FileSystem.FileSystem;
83 const path = yield* Path.Path;
84 const runner = yield* ProcessRunner.ProcessRunner;
85 const home = process.env.HOME;
86 if (!home) {
87 return yield* Effect.die(new Error("HOME is required for a systemd user-manager test."));
88 }
89
90 const directory = yield* fs.makeTempDirectoryScoped({ prefix: `t3-systemd-${scenario}-` });
91 const runtimeStatePath = path.join(directory, "server-runtime.json");
92 const launchesPath = path.join(directory, "launches.jsonl");
93 const receiptPath = path.join(directory, "receipt.json");
94 const planPath = path.join(directory, "plan.json");
95 const fixtureServerPath = path.join(directory, "fixture-server.mjs");
96 yield* fs.writeFileString(fixtureServerPath, fixtureServerSource);
97
98 const suffix = `${String(process.pid)}-${String(yield* Clock.currentTimeMillis)}-${scenario}`;
99 const serviceUnit = `t3code-self-update-acceptance-${suffix}.service`;
100 const helperUnit = `t3code-self-update-acceptance-helper-${suffix}.service`;
101 const userUnitDirectory = path.join(home, ".config", "systemd", "user");
102 const unitPath = path.join(userUnitDirectory, serviceUnit);
103 yield* fs.makeDirectory(userUnitDirectory, { recursive: true });
104
105 const renderUnit = (version: string, readiness: "ready" | "blocked") =>
106 [
107 "[Unit]",
108 "Description=T3 self-update acceptance fixture",
109 "",
110 "[Service]",
111 "Type=simple",
112 `ExecStart=${quoteSystemdValue(process.execPath)} ${quoteSystemdValue(fixtureServerPath)} ${quoteSystemdValue(runtimeStatePath)} ${quoteSystemdValue(launchesPath)} ${version} ${readiness}`,
113 "Restart=no",
114 "TimeoutStopSec=5",
115 "",
116 ].join("\n");
117 const previousUnit = renderUnit(OLD_VERSION, "ready");
118 const nextUnit = renderUnit(TARGET_VERSION, scenario === "healthy" ? "ready" : "blocked");
119
120 const systemctl = (args: ReadonlyArray<string>) =>
121 runner.run({ command: "systemctl", args: ["--user", ...args] });
122 const cleanup = Effect.gen(function* () {
123 yield* systemctl(["stop", helperUnit, serviceUnit]).pipe(Effect.ignore);
124 yield* systemctl(["reset-failed", helperUnit, serviceUnit]).pipe(Effect.ignore);
125 yield* fs.remove(unitPath, { force: true }).pipe(Effect.ignore);
126 yield* systemctl(["daemon-reload"]).pipe(Effect.ignore);
127 });
128 yield* Effect.addFinalizer(() => cleanup);
129
130 yield* fs.writeFileString(unitPath, previousUnit);
131 const reload = yield* systemctl(["daemon-reload"]);
132 assert.equal(Number(reload.code), 0);
133 const start = yield* systemctl(["start", serviceUnit]);
134 assert.equal(Number(start.code), 0);
135 yield* waitForSystemdSelfUpdateReadiness(
136 {
137 expectedVersion: OLD_VERSION,
138 previousPid: 0,
139 runtimeStatePath,
140 },
141 { timeout: Duration.seconds(10), interval: Duration.millis(100) },
142 );
143 const initialState = yield* readPersistedServerRuntimeState(runtimeStatePath);
144 if (Option.isNone(initialState)) {
145 return yield* Effect.die(
146 new Error("The disposable systemd service did not publish runtime state."),
147 );
148 }
149
150 const plan = new SystemdSelfUpdatePlan({
151 version: 1,
152 fromVersion: OLD_VERSION,
153 targetVersion: TARGET_VERSION,
154 currentPid: initialState.value.pid,
155 serviceUnit,
156 unitPath,
157 previousUnit,
158 nextUnit,
159 runtimeStatePath,
160 receiptPath,
161 });
162 yield* writeSystemdSelfUpdatePlan(planPath, plan);
163
164 const repositoryRoot = path.resolve(import.meta.dirname, "../../../..");
165 const cliEntryPath = path.join(repositoryRoot, "apps", "server", "src", "bin.ts");
166 const handoff = yield* runner.run({
167 command: "systemd-run",
168 args: [
169 "--user",
170 "--wait",
171 "--collect",
172 "--service-type=exec",
173 `--unit=${helperUnit}`,
174 process.execPath,
175 "--experimental-strip-types",
176 cliEntryPath,
177 "service",
178 "_apply-update",
179 "--plan",
180 planPath,
181 ],
182 timeout: Duration.seconds(90),
183 });
184 assert.equal(Number(handoff.code), 0, handoff.stderr);
185
186 const receipt = yield* fs
187 .readFileString(receiptPath)
188 .pipe(
189 Effect.flatMap(Schema.decodeUnknownEffect(Schema.fromJsonString(SystemdSelfUpdateReceipt))),
190 );
191 assert.equal(receipt.phase, scenario === "healthy" ? "healthy" : "rolled-back");
192 assert.equal(
193 yield* fs.readFileString(unitPath),
194 scenario === "healthy" ? nextUnit : previousUnit,
195 );
196
197 yield* waitForSystemdSelfUpdateReadiness(
198 {
199 expectedVersion: scenario === "healthy" ? TARGET_VERSION : OLD_VERSION,
200 previousPid: initialState.value.pid,
201 runtimeStatePath,
202 },
203 { timeout: Duration.seconds(10), interval: Duration.millis(100) },
204 );
205 const launches = (yield* fs.readFileString(launchesPath))
206 .trim()
207 .split("\n")
208 .map((line) => decodeLaunch(line));
209 assert.deepEqual(
210 launches.map(({ version }) => version),
211 scenario === "healthy"
212 ? [OLD_VERSION, TARGET_VERSION]
213 : [OLD_VERSION, TARGET_VERSION, OLD_VERSION],
214 );
215});
216
217describe.runIf(process.env.T3_SYSTEMD_SELF_UPDATE_ACCEPTANCE === "1")(
218 "systemd self-update acceptance",
219 () => {
220 const layer = ProcessRunner.layer.pipe(
221 Layer.provideMerge(NodeServices.layer),
222 Layer.merge(FetchHttpClient.layer),
223 );
224
225 it.live(
226 "keeps the helper alive while replacing the host service",
227 () => runAcceptanceScenario("healthy").pipe(Effect.provide(layer)),
228 30_000,
229 );
230
231 it.live(
232 "rolls back a server that binds HTTP without becoming runtime-ready",
233 () => runAcceptanceScenario("rollback").pipe(Effect.provide(layer)),
234 120_000,
235 );
236 },
237);
apps/server/src/cloud/systemdSelfUpdate.test.ts+56−4
@@ -6,14 +6,17 @@ import * as FileSystem from "effect/FileSystem"; · 5 unmodified lines
6import * as Layer from "effect/Layer";6import * as Layer from "effect/Layer";
7import * as Path from "effect/Path";7import * as Path from "effect/Path";
8import * as Schema from "effect/Schema";8import * as Schema from "effect/Schema";
9import { HttpClient, HttpClientResponse } from "effect/unstable/http";
9import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";10import * as ChildProcessSpawner from "effect/unstable/process/ChildProcessSpawner";
10 11
11import * as ProcessRunner from "../processRunner.ts";12import * as ProcessRunner from "../processRunner.ts";
13import { PersistedServerRuntimeState } from "../serverRuntimeState.ts";
12import {14import {
13 SystemdSelfUpdateActivationError,15 SystemdSelfUpdateActivationError,
14 SystemdSelfUpdatePlan,16 SystemdSelfUpdatePlan,
15 SystemdSelfUpdateReceipt,17 SystemdSelfUpdateReceipt,
16 activateSystemdSelfUpdatePlan,18 activateSystemdSelfUpdatePlan,
19 waitForSystemdSelfUpdateReadiness,
17} from "./systemdSelfUpdate.ts";20} from "./systemdSelfUpdate.ts";
18 21
19interface RecordedCommand {22interface RecordedCommand {
@@ -59,6 +62,7 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { · 39 unmodified lines
59 fromVersion: "0.0.28",62 fromVersion: "0.0.28",
60 targetVersion: "0.0.29",63 targetVersion: "0.0.29",
61 currentPid: 123,64 currentPid: 123,
65 serviceUnit: "t3code-self-update-test.service",
62 unitPath,66 unitPath,
63 previousUnit,67 previousUnit,
64 nextUnit: "new unit\n",68 nextUnit: "new unit\n",
@@ -95,7 +99,7 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { · 30 unmodified lines
95 commands.map(({ args }) => args),99 commands.map(({ args }) => args),
96 [100 [
97 ["--user", "daemon-reload"],101 ["--user", "daemon-reload"],
98 ["--user", "restart", "t3code.service"],102 ["--user", "restart", "t3code-self-update-test.service"],
99 ],103 ],
100 );104 );
101 assert.equal((yield* readReceipt(fixture.receiptPath)).phase, "healthy");105 assert.equal((yield* readReceipt(fixture.receiptPath)).phase, "healthy");
@@ -132,10 +136,10 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { · 30 unmodified lines
132 commands.map(({ args }) => args),136 commands.map(({ args }) => args),
133 [137 [
134 ["--user", "daemon-reload"],138 ["--user", "daemon-reload"],
135 ["--user", "restart", "t3code.service"],139 ["--user", "restart", "t3code-self-update-test.service"],
136 ["--user", "daemon-reload"],140 ["--user", "daemon-reload"],
137 ["--user", "reset-failed", "t3code.service"],141 ["--user", "reset-failed", "t3code-self-update-test.service"],
138 ["--user", "restart", "t3code.service"],142 ["--user", "restart", "t3code-self-update-test.service"],
139 ],143 ],
140 );144 );
141 const receipt = yield* readReceipt(fixture.receiptPath);145 const receipt = yield* readReceipt(fixture.receiptPath);
@@ -164,4 +168,52 @@ it.layer(NodeServices.layer)("systemd self-update activation", (it) => { · 22 unmodified lines
164 assert.equal((yield* readReceipt(fixture.receiptPath)).phase, "recovery-failed");168 assert.equal((yield* readReceipt(fixture.receiptPath)).phase, "recovery-failed");
165 }),169 }),
166 );170 );
171
172 it.effect("probes the runtime-readiness endpoint on the replacement process", () =>
173 Effect.gen(function* () {
174 const fs = yield* FileSystem.FileSystem;
175 const path = yield* Path.Path;
176 const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-systemd-ready-test-" });
177 const runtimeStatePath = path.join(directory, "server-runtime.json");
178 const encodeRuntimeState = Schema.encodeEffect(
179 Schema.fromJsonString(PersistedServerRuntimeState),
180 );
181 const encodedRuntimeState = yield* encodeRuntimeState({
182 version: 1,
183 pid: 456,
184 port: 3210,
185 origin: "http://127.0.0.1:3210",
186 startedAt: "2026-01-01T00:00:00.000Z",
187 });
188 yield* fs.writeFileString(runtimeStatePath, `${encodedRuntimeState}\n`);
189
190 const requests: Array<string> = [];
191 const httpClientLayer = Layer.succeed(
192 HttpClient.HttpClient,
193 HttpClient.make((request) => {
194 requests.push(request.url);
195 return Effect.succeed(
196 HttpClientResponse.fromWeb(
197 request,
198 Response.json({
199 environmentId: "environment-test",
200 label: "Test environment",
201 platform: { os: "linux", arch: "x64" },
202 serverVersion: "0.0.29",
203 capabilities: { repositoryIdentity: true },
204 }),
205 ),
206 );
207 }),
208 );
209
210 yield* waitForSystemdSelfUpdateReadiness({
211 expectedVersion: "0.0.29",
212 previousPid: 123,
213 runtimeStatePath,
214 }).pipe(Effect.provide(httpClientLayer));
215
216 assert.deepEqual(requests, ["http://127.0.0.1:3210/.well-known/t3/ready"]);
217 }),
218 );
167});219});
apps/server/src/cloud/systemdSelfUpdate.ts+8−7
@@ -12,7 +12,6 @@ import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstab · 11 unmodified lines
12import { writeFileStringAtomically } from "../atomicWrite.ts";12import { writeFileStringAtomically } from "../atomicWrite.ts";
13import * as ProcessRunner from "../processRunner.ts";13import * as ProcessRunner from "../processRunner.ts";
14import { readPersistedServerRuntimeState } from "../serverRuntimeState.ts";14import { readPersistedServerRuntimeState } from "../serverRuntimeState.ts";
15import { BOOT_SERVICE_UNIT_FILE } from "./bootService.ts";
16 15
17export const SYSTEMD_SELF_UPDATE_UNIT = "t3code-self-update.service";16export const SYSTEMD_SELF_UPDATE_UNIT = "t3code-self-update.service";
18export const SYSTEMD_SELF_UPDATE_DIRECTORY = "self-update";17export const SYSTEMD_SELF_UPDATE_DIRECTORY = "self-update";
@@ -23,7 +22,7 @@ const DEFAULT_RESTART_DELAY = Duration.seconds(2); · 4 unmodified lines
23const DEFAULT_READINESS_TIMEOUT = Duration.seconds(60);22const DEFAULT_READINESS_TIMEOUT = Duration.seconds(60);
24const DEFAULT_READINESS_INTERVAL = Duration.millis(500);23const DEFAULT_READINESS_INTERVAL = Duration.millis(500);
25const READINESS_REQUEST_TIMEOUT = Duration.seconds(2);24const READINESS_REQUEST_TIMEOUT = Duration.seconds(2);
26const WELL_KNOWN_ENVIRONMENT_PATH = "/.well-known/t3/environment";25const WELL_KNOWN_READY_PATH = "/.well-known/t3/ready";
27 26
28export class SystemdSelfUpdatePlan extends Schema.Class<SystemdSelfUpdatePlan>(27export class SystemdSelfUpdatePlan extends Schema.Class<SystemdSelfUpdatePlan>(
29 "SystemdSelfUpdatePlan",28 "SystemdSelfUpdatePlan",
@@ -32,6 +31,7 @@ export class SystemdSelfUpdatePlan extends Schema.Class<SystemdSelfUpdatePlan>( · 2 unmodified lines
32 fromVersion: Schema.String,31 fromVersion: Schema.String,
33 targetVersion: Schema.String,32 targetVersion: Schema.String,
34 currentPid: Schema.Int,33 currentPid: Schema.Int,
34 serviceUnit: Schema.String,
35 unitPath: Schema.String,35 unitPath: Schema.String,
36 previousUnit: Schema.String,36 previousUnit: Schema.String,
37 nextUnit: Schema.String,37 nextUnit: Schema.String,
@@ -185,7 +185,8 @@ const installUnit = Effect.fn("systemdSelfUpdate.installUnit")(function* ( · 147 unmodified lines
185 yield* runSystemctl(["--user", "daemon-reload"], "daemon-reload");185 yield* runSystemctl(["--user", "daemon-reload"], "daemon-reload");
186});186});
187 187
188const restartBootService = runSystemctl(["--user", "restart", BOOT_SERVICE_UNIT_FILE], "restart");188const restartBootService = (serviceUnit: string) =>
189 runSystemctl(["--user", "restart", serviceUnit], "restart");
189 190
190export interface SystemdSelfUpdateReadinessInput {191export interface SystemdSelfUpdateReadinessInput {
191 readonly expectedVersion: string;192 readonly expectedVersion: string;
@@ -227,7 +228,7 @@ export const waitForSystemdSelfUpdateReadiness = Effect.fn("systemdSelfUpdate.wa · 35 unmodified lines
227 }228 }
228 229
229 const request = HttpClientRequest.get(230 const request = HttpClientRequest.get(
230 new URL(WELL_KNOWN_ENVIRONMENT_PATH, state.value.origin).toString(),231 new URL(WELL_KNOWN_READY_PATH, state.value.origin).toString(),
231 );232 );
232 const descriptor = yield* client.execute(request).pipe(233 const descriptor = yield* client.execute(request).pipe(
233 Effect.timeout(READINESS_REQUEST_TIMEOUT),234 Effect.timeout(READINESS_REQUEST_TIMEOUT),
@@ -277,7 +278,7 @@ export const activateSystemdSelfUpdatePlan = Effect.fn("systemdSelfUpdate.activa · 43 unmodified lines
277 yield* writeReceipt(plan, "activating").pipe(Effect.ignore);278 yield* writeReceipt(plan, "activating").pipe(Effect.ignore);
278 yield* Effect.sleep(options.restartDelay ?? DEFAULT_RESTART_DELAY);279 yield* Effect.sleep(options.restartDelay ?? DEFAULT_RESTART_DELAY);
279 yield* installUnit(plan, plan.nextUnit);280 yield* installUnit(plan, plan.nextUnit);
280 yield* restartBootService;281 yield* restartBootService(plan.serviceUnit);
281 yield* waitForReadiness({282 yield* waitForReadiness({
282 expectedVersion: plan.targetVersion,283 expectedVersion: plan.targetVersion,
283 previousPid: plan.currentPid,284 previousPid: plan.currentPid,
@@ -298,8 +299,8 @@ export const activateSystemdSelfUpdatePlan = Effect.fn("systemdSelfUpdate.activa · 14 unmodified lines
298 );299 );
299 const recovery = Effect.gen(function* () {300 const recovery = Effect.gen(function* () {
300 yield* installUnit(plan, plan.previousUnit);301 yield* installUnit(plan, plan.previousUnit);
301 yield* runSystemctl(["--user", "reset-failed", BOOT_SERVICE_UNIT_FILE], "reset-failed");302 yield* runSystemctl(["--user", "reset-failed", plan.serviceUnit], "reset-failed");
302 yield* restartBootService;303 yield* restartBootService(plan.serviceUnit);
303 yield* waitForReadiness({304 yield* waitForReadiness({
304 expectedVersion: plan.fromVersion,305 expectedVersion: plan.fromVersion,
305 previousPid: plan.currentPid,306 previousPid: plan.currentPid,
apps/server/src/http.ts+18−7
@@ -40,6 +40,7 @@ import { · 39 unmodified lines
40} from "./auth/http.ts";40} from "./auth/http.ts";
41import * as ServerEnvironment from "./environment/ServerEnvironment.ts";41import * as ServerEnvironment from "./environment/ServerEnvironment.ts";
42import { browserApiCorsAllowedHeaders, browserApiCorsAllowedMethods } from "./httpCors.ts";42import { browserApiCorsAllowedHeaders, browserApiCorsAllowedMethods } from "./httpCors.ts";
43import * as ServerRuntimeStartup from "./serverRuntimeStartup.ts";
43 44
44const OTLP_TRACES_PROXY_PATH = "/api/observability/v1/traces";45const OTLP_TRACES_PROXY_PATH = "/api/observability/v1/traces";
45const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]);46const LOOPBACK_HOSTNAMES = new Set(["127.0.0.1", "::1", "localhost"]);
@@ -177,13 +178,23 @@ export const serverEnvironmentHttpApiLayer = HttpApiBuilder.group( · 131 unmodified lines
177 "metadata",178 "metadata",
178 Effect.fnUntraced(function* (handlers) {179 Effect.fnUntraced(function* (handlers) {
179 const serverEnvironment = yield* ServerEnvironment.ServerEnvironment;180 const serverEnvironment = yield* ServerEnvironment.ServerEnvironment;
180 return handlers.handle(181 const startup = yield* ServerRuntimeStartup.ServerRuntimeStartup;
181 "descriptor",182 return handlers
182 Effect.fn("environment.metadata.descriptor")(function* (args) {183 .handle(
183 yield* annotateEnvironmentRequest(args.endpoint.name);184 "descriptor",
184 return yield* serverEnvironment.getDescriptor;185 Effect.fn("environment.metadata.descriptor")(function* (args) {
185 }, traceRelayRequest),186 yield* annotateEnvironmentRequest(args.endpoint.name);
186 );187 return yield* serverEnvironment.getDescriptor;
188 }, traceRelayRequest),
189 )
190 .handle(
191 "ready",
192 Effect.fn("environment.metadata.ready")(function* (args) {
193 yield* annotateEnvironmentRequest(args.endpoint.name);
194 yield* startup.awaitReady.pipe(Effect.orDie);
195 return yield* serverEnvironment.getDescriptor;
196 }, traceRelayRequest),
197 );
187 }),198 }),
188);199);
189 200
apps/server/src/persistence/Migrations.test.ts+51−1
@@ -3,7 +3,12 @@ import * as Effect from "effect/Effect"; · 2 unmodified lines
3import * as Layer from "effect/Layer";3import * as Layer from "effect/Layer";
4import * as SqlClient from "effect/unstable/sql/SqlClient";4import * as SqlClient from "effect/unstable/sql/SqlClient";
5 5
6import { MigrationIdentityMismatchError, validateMigrationIdentities } from "./Migrations.ts";6import {
7 AutomaticUpdateMigrationFrontierError,
8 MigrationIdentityMismatchError,
9 validateAutomaticUpdateMigrationFrontier,
10 validateMigrationIdentities,
11} from "./Migrations.ts";
7import * as NodeSqliteClient from "./NodeSqliteClient.ts";12import * as NodeSqliteClient from "./NodeSqliteClient.ts";
8 13
9const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory()));14const layer = it.layer(Layer.mergeAll(NodeSqliteClient.layerMemory()));
@@ -50,4 +55,49 @@ layer("migration identity validation", (it) => { · 40 unmodified lines
50 yield* validateMigrationIdentities();55 yield* validateMigrationIdentities();
51 }),56 }),
52 );57 );
58
59 it.effect("rejects automatic downgrade when the database has a newer migration", () =>
60 Effect.gen(function* () {
61 const sql = yield* SqlClient.SqlClient;
62 yield* createTrackingTable();
63 yield* sql`
64 INSERT INTO effect_sql_migrations (migration_id, name)
65 VALUES (999, 'FutureMigration')
66 `;
67
68 const error = yield* validateAutomaticUpdateMigrationFrontier().pipe(Effect.flip);
69 assert.instanceOf(error, AutomaticUpdateMigrationFrontierError);
70 assert.equal(error.databaseMigrationId, 999);
71 assert.equal(error.targetMigrationId, 35);
72 }),
73 );
74
75 it.effect("rejects automatic updates that would run migrations", () =>
76 Effect.gen(function* () {
77 const sql = yield* SqlClient.SqlClient;
78 yield* createTrackingTable();
79 yield* sql`
80 INSERT INTO effect_sql_migrations (migration_id, name)
81 VALUES (34, 'ProjectionThreadsSnoozed')
82 `;
83
84 const error = yield* validateAutomaticUpdateMigrationFrontier().pipe(Effect.flip);
85 assert.instanceOf(error, AutomaticUpdateMigrationFrontierError);
86 assert.equal(error.databaseMigrationId, 34);
87 assert.equal(error.targetMigrationId, 35);
88 }),
89 );
90
91 it.effect("allows automatic updates on the same migration frontier", () =>
92 Effect.gen(function* () {
93 const sql = yield* SqlClient.SqlClient;
94 yield* createTrackingTable();
95 yield* sql`
96 INSERT INTO effect_sql_migrations (migration_id, name)
97 VALUES (35, 'ProjectionThreadTitleRegeneration')
98 `;
99
100 yield* validateAutomaticUpdateMigrationFrontier();
101 }),
102 );
53});103});
apps/server/src/persistence/Migrations.ts+58−18
@@ -127,33 +127,53 @@ export class MigrationIdentityMismatchError extends Schema.TaggedErrorClass<Migr · 126 unmodified lines
127 }127 }
128}128}
129 129
130export class AutomaticUpdateMigrationFrontierError extends Schema.TaggedErrorClass<AutomaticUpdateMigrationFrontierError>()(
131 "AutomaticUpdateMigrationFrontierError",
132 {
133 databaseMigrationId: Schema.Int,
134 targetMigrationId: Schema.Int,
135 },
136) {
137 override get message(): string {
138 return this.databaseMigrationId > this.targetMigrationId
139 ? `This database is newer than the requested t3 build (migration ${String(this.databaseMigrationId)} vs ${String(this.targetMigrationId)}).`
140 : `The requested t3 build would migrate this database (migration ${String(this.databaseMigrationId)} to ${String(this.targetMigrationId)}), which is not safe during an automatic update.`;
141 }
142}
143
144const readPersistedMigrationIdentities = Effect.fn("readPersistedMigrationIdentities")(
145 function* () {
146 const sql = yield* SqlClient.SqlClient;
147 const tables = yield* sql<{ readonly name: string }>`
148 SELECT name
149 FROM sqlite_master
150 WHERE type = 'table' AND name = 'effect_sql_migrations'
151 `;
152 if (tables.length === 0) {
153 return [];
154 }
155
156 return yield* sql<{
157 readonly migration_id: number;
158 readonly name: string;
159 }>`
160 SELECT migration_id, name
161 FROM effect_sql_migrations
162 ORDER BY migration_id
163 `;
164 },
165);
166
130/**167/**
131 * Refuse to run a build whose migration history disagrees with the database.168 * Refuse to run a build whose migration history disagrees with the database.
132 * Unknown newer IDs are intentionally allowed so a compatible older build can169 * Unknown newer IDs are intentionally allowed so a compatible older build can
133 * inspect a database created by a newer release without rewriting its history.170 * inspect a database created by a newer release without rewriting its history.
134 */171 */
135export const validateMigrationIdentities = Effect.fn("validateMigrationIdentities")(function* () {172export const validateMigrationIdentities = Effect.fn("validateMigrationIdentities")(function* () {
136 const sql = yield* SqlClient.SqlClient;
137 const tables = yield* sql<{ readonly name: string }>`
138 SELECT name
139 FROM sqlite_master
140 WHERE type = 'table' AND name = 'effect_sql_migrations'
141 `;
142 if (tables.length === 0) {
143 return;
144 }
145
146 const expectedNames = new Map<number, string>(173 const expectedNames = new Map<number, string>(
147 migrationEntries.map(([migrationId, name]) => [migrationId, name]),174 migrationEntries.map(([migrationId, name]) => [migrationId, name]),
148 );175 );
149 const persisted = yield* sql<{176 const persisted = yield* readPersistedMigrationIdentities();
150 readonly migration_id: number;
151 readonly name: string;
152 }>`
153 SELECT migration_id, name
154 FROM effect_sql_migrations
155 ORDER BY migration_id
156 `;
157 177
158 for (const migration of persisted) {178 for (const migration of persisted) {
159 const expectedName = expectedNames.get(migration.migration_id);179 const expectedName = expectedNames.get(migration.migration_id);
@@ -167,6 +187,26 @@ export const validateMigrationIdentities = Effect.fn("validateMigrationIdentitie · 7 unmodified lines
167 }187 }
168});188});
169 189
190/**
191 * Automatic activation is safe only when the target and database are already
192 * on the same schema. A binary rollback cannot undo a migration, and an older
193 * target cannot safely interpret migration IDs that it does not know about.
194 */
195export const validateAutomaticUpdateMigrationFrontier = Effect.fn(
196 "validateAutomaticUpdateMigrationFrontier",
197)(function* () {
198 yield* validateMigrationIdentities();
199 const persisted = yield* readPersistedMigrationIdentities();
200 const databaseMigrationId = persisted.at(-1)?.migration_id ?? 0;
201 const targetMigrationId = migrationEntries.at(-1)?.[0] ?? 0;
202 if (databaseMigrationId !== targetMigrationId) {
203 return yield* new AutomaticUpdateMigrationFrontierError({
204 databaseMigrationId,
205 targetMigrationId,
206 });
207 }
208});
209
170export interface RunMigrationsOptions {210export interface RunMigrationsOptions {
171 readonly toMigrationInclusive?: number | undefined;211 readonly toMigrationInclusive?: number | undefined;
172}212}
apps/server/src/server.test.ts+25−0
@@ -782,6 +782,7 @@ const buildAppUnderTest = (options?: { · 781 unmodified lines
782 Layer.provide(782 Layer.provide(
783 Layer.mock(ServerRuntimeStartup.ServerRuntimeStartup)({783 Layer.mock(ServerRuntimeStartup.ServerRuntimeStartup)({
784 awaitCommandReady: Effect.void,784 awaitCommandReady: Effect.void,
785 awaitReady: Effect.void,
785 markHttpListening: Effect.void,786 markHttpListening: Effect.void,
786 enqueueCommand: (effect) => effect,787 enqueueCommand: (effect) => effect,
787 ...options?.layers?.serverRuntimeStartup,788 ...options?.layers?.serverRuntimeStartup,
@@ -1361,6 +1362,30 @@ it.layer(NodeServices.layer)("server router seam", (it) => { · 573 unmodified lines
1361 }).pipe(Effect.provide(NodeHttpServer.layerTest)),1362 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
1362 );1363 );
1363 1364
1365 it.effect("does not report update readiness before the runtime publishes ready", () =>
1366 Effect.gen(function* () {
1367 const runtimeReady = yield* Deferred.make<void>();
1368 yield* buildAppUnderTest({
1369 layers: {
1370 serverRuntimeStartup: {
1371 awaitReady: Deferred.await(runtimeReady),
1372 },
1373 },
1374 });
1375
1376 const url = yield* getHttpServerUrl("/.well-known/t3/ready");
1377 const responseFiber = yield* fetchEffect(url).pipe(Effect.forkScoped);
1378 yield* Effect.yieldNow;
1379 assert.isUndefined(responseFiber.pollUnsafe());
1380
1381 yield* Deferred.succeed(runtimeReady, undefined);
1382 const response = yield* Fiber.join(responseFiber);
1383 const body = yield* responseJsonEffect<typeof testEnvironmentDescriptor>(response);
1384 assert.equal(response.status, 200);
1385 assert.deepEqual(body, testEnvironmentDescriptor);
1386 }).pipe(Effect.provide(NodeHttpServer.layerTest)),
1387 );
1388
1364 it.effect("compresses large JSON responses through the composed routes", () =>1389 it.effect("compresses large JSON responses through the composed routes", () =>
1365 Effect.gen(function* () {1390 Effect.gen(function* () {
1366 const descriptor = {1391 const descriptor = {
apps/server/src/serverRuntimeStartup.ts+5−0
@@ -59,6 +59,7 @@ export class ServerRuntimeStartup extends Context.Service< · 58 unmodified lines
59 ServerRuntimeStartup,59 ServerRuntimeStartup,
60 {60 {
61 readonly awaitCommandReady: Effect.Effect<void, ServerRuntimeStartupError>;61 readonly awaitCommandReady: Effect.Effect<void, ServerRuntimeStartupError>;
62 readonly awaitReady: Effect.Effect<void, ServerRuntimeStartupError>;
62 readonly markHttpListening: Effect.Effect<void>;63 readonly markHttpListening: Effect.Effect<void>;
63 readonly enqueueCommand: <A, E>(64 readonly enqueueCommand: <A, E>(
64 effect: Effect.Effect<A, E>,65 effect: Effect.Effect<A, E>,
@@ -300,6 +301,7 @@ export const make = Effect.gen(function* () { · 235 unmodified lines
300 301
301 const commandGate = yield* makeCommandGate;302 const commandGate = yield* makeCommandGate;
302 const httpListening = yield* Deferred.make<void>();303 const httpListening = yield* Deferred.make<void>();
304 const runtimeReady = yield* Deferred.make<void, ServerRuntimeStartupError>();
303 const reactorScope = yield* Scope.make("sequential");305 const reactorScope = yield* Scope.make("sequential");
304 306
305 yield* Effect.addFinalizer(() => Scope.close(reactorScope, Exit.void));307 yield* Effect.addFinalizer(() => Scope.close(reactorScope, Exit.void));
@@ -425,6 +427,7 @@ export const make = Effect.gen(function* () { · 119 unmodified lines
425 });427 });
426 yield* Effect.logError("server runtime startup failed", { cause: startupExit.cause });428 yield* Effect.logError("server runtime startup failed", { cause: startupExit.cause });
427 yield* commandGate.failCommandReady(error);429 yield* commandGate.failCommandReady(error);
430 yield* Deferred.fail(runtimeReady, error).pipe(Effect.orDie);
428 return;431 return;
429 }432 }
430 433
@@ -444,6 +447,7 @@ export const make = Effect.gen(function* () { · 13 unmodified lines
444 },447 },
445 }),448 }),
446 );449 );
450 yield* Deferred.succeed(runtimeReady, undefined).pipe(Effect.orDie);
447 451
448 yield* Effect.logDebug("startup phase: recording startup heartbeat");452 yield* Effect.logDebug("startup phase: recording startup heartbeat");
449 yield* launchStartupHeartbeat;453 yield* launchStartupHeartbeat;
@@ -470,6 +474,7 @@ export const make = Effect.gen(function* () { · 20 unmodified lines
470 474
471 return {475 return {
472 awaitCommandReady: commandGate.awaitCommandReady,476 awaitCommandReady: commandGate.awaitCommandReady,
477 awaitReady: Deferred.await(runtimeReady),
473 markHttpListening: Deferred.succeed(httpListening, undefined),478 markHttpListening: Deferred.succeed(httpListening, undefined),
474 enqueueCommand: commandGate.enqueueCommand,479 enqueueCommand: commandGate.enqueueCommand,
475 } satisfies ServerRuntimeStartup["Service"];480 } satisfies ServerRuntimeStartup["Service"];
docs/internals/server-updates.md+9−4
@@ -90,9 +90,12 @@ successfully. Boot-service setup and self-update share the same process-wide ins · 89 unmodified lines
90they cannot mutate a pinned runtime concurrently.90they cannot mutate a pinned runtime concurrently.
91 91
92Before any restart, the current Node executable runs the replacement with `--version`, then asks92Before any restart, the current Node executable runs the replacement with `--version`, then asks
93the target artifact to validate its migration identities against the live database in read-only93the target artifact to validate the live database in read-only mode. Automatic activation requires
94mode. A failed install, version preflight, or compatibility check leaves the current server running.94the target and database to be on the exact same migration frontier. This rejects both downgrades and
95Targets old enough to lack the compatibility command are not eligible for remote activation.95updates that would migrate the database because a binary rollback cannot undo a schema change. Use
96the manual host service update path for those releases. A failed install, version preflight, or
97compatibility check leaves the current server running. Targets old enough to lack the compatibility
98command are not eligible for remote activation.
96 99
97Candidate cleanup is narrower than "any failed preflight". The candidate runtime is removed only when100Candidate cleanup is narrower than "any failed preflight". The candidate runtime is removed only when
98the preflight process actually completes and reports a bad exit code or the wrong version: that is101the preflight process actually completes and reports a bad exit code or the wrong version: that is
@@ -118,7 +121,9 @@ For `boot-service`, the request process writes a typed plan but leaves its live · 19 unmodified lines
118starts a fixed-name `t3code-self-update.service` transient unit with `systemd-run --user`; the121starts a fixed-name `t3code-self-update.service` transient unit with `systemd-run --user`; the
119transient unit has a separate cgroup and acts as the cross-process update lock. After a short grace122transient unit has a separate cgroup and acts as the cross-process update lock. After a short grace
120period it atomically installs the candidate unit, reloads systemd, performs a blocking restart, and123period it atomically installs the candidate unit, reloads systemd, performs a blocking restart, and
121probes the public environment descriptor for the exact target version. If activation or readiness124probes the public runtime-readiness endpoint for the exact target version. The endpoint does not
125respond until the server's command runtime is usable and its lifecycle `ready` event has been
126published, so an HTTP listener alone cannot make an activation healthy. If activation or readiness
122fails, the helper restores the previous unit, clears systemd's failed state, restarts it, and proves127fails, the helper restores the previous unit, clears systemd's failed state, restarts it, and proves
123the previous version is ready. The plan and latest receipt live under the environment state128the previous version is ready. The plan and latest receipt live under the environment state
124directory's `self-update` folder for diagnosis.129directory's `self-update` folder for diagnosis.
docs/operations/release.md+17−0
@@ -190,6 +190,23 @@ connect the new client to a server on the previous version and verify that the u · 189 unmodified lines
190reconnects to the matching server. Test one automatic path and the manual or desktop-managed190reconnects to the matching server. Test one automatic path and the manual or desktop-managed
191guidance when those environments are available.191guidance when those environments are available.
192 192
193The automatic boot-service path also requires a real systemd user-manager acceptance run. Use a
194disposable T3 home and uniquely named host and helper units, never the developer's daily
195`t3code.service`. Exercise both outcomes:
196
1971. A same-migration-frontier candidate becomes runtime-ready, the helper survives the host-unit
198 restart, and the receipt reaches `healthy` with a new runtime PID and the exact target version.
1992. A candidate that binds HTTP but never becomes runtime-ready times out, the previous unit is
200 restored, and the receipt reaches `rolled-back` with the old version command-ready again.
201
202Treat the systemd run as a release blocker when the update lifecycle changes. CI jobs without a
203user manager may skip it, but a Linux release host must record the run before publishing.
204
205```bash
206T3_SYSTEMD_SELF_UPDATE_ACCEPTANCE=1 vp test run \
207 apps/server/src/cloud/systemdSelfUpdate.acceptance.test.ts
208```
209
193## Desktop auto-update notes210## Desktop auto-update notes
194 211
195- Updater runtime: `apps/desktop/src/updates/DesktopUpdates.ts`.212- Updater runtime: `apps/desktop/src/updates/DesktopUpdates.ts`.
packages/contracts/src/environmentHttp.ts+11−5
@@ -371,11 +371,17 @@ export const AuthOtherClientSessionsRevokeResult = Schema.Struct({ · 370 unmodified lines
371});371});
372export type AuthOtherClientSessionsRevokeResult = typeof AuthOtherClientSessionsRevokeResult.Type;372export type AuthOtherClientSessionsRevokeResult = typeof AuthOtherClientSessionsRevokeResult.Type;
373 373
374export class EnvironmentMetadataHttpApi extends HttpApiGroup.make("metadata").add(374export class EnvironmentMetadataHttpApi extends HttpApiGroup.make("metadata")
375 HttpApiEndpoint.get("descriptor", "/.well-known/t3/environment", {375 .add(
376 success: ExecutionEnvironmentDescriptor,376 HttpApiEndpoint.get("descriptor", "/.well-known/t3/environment", {
377 }),377 success: ExecutionEnvironmentDescriptor,
378) {}378 }),
379 )
380 .add(
381 HttpApiEndpoint.get("ready", "/.well-known/t3/ready", {
382 success: ExecutionEnvironmentDescriptor,
383 }),
384 ) {}
379 385
380export class EnvironmentAuthHttpApi extends HttpApiGroup.make("auth")386export class EnvironmentAuthHttpApi extends HttpApiGroup.make("auth")
381 .add(387 .add(

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t