fix(server): release the tunnel when an explicit stop interrupts an update

A pending update in the launcher state is not proof a replacement server is coming: `t3 service uninstall` or `systemctl stop` during the pending window also tears the server down, permanently. The launcher now writes a stop marker before signalling its child on an explicit stop and clears it on the next start; the shutdown tunnel release keeps the tunnel only for pending updates without the marker.

Theo Browne committed 3 months ago

Commit f9e82361 parent 8f341f25 files · +100 −5Browse files

Unified · Split

5 files

apps/server/src/cloud/http.test.ts +31 −0
apps/server/src/cloud/http.ts +22 −5
apps/server/src/cloud/serviceProtocol.ts +4 −0
apps/server/src/serviceLauncher.test.ts +26 −0
apps/server/src/serviceLauncher.ts +17 −0

apps/server/src/cloud/http.test.ts+31−0

@@ -1,6 +1,7 @@
1	import * as NodeServices from "@effect/platform-node/NodeServices";	1	import * as NodeServices from "@effect/platform-node/NodeServices";
2	import { describe, expect, it } from "@effect/vitest";	2	import { describe, expect, it } from "@effect/vitest";
3	import * as Effect from "effect/Effect";	3	import * as Effect from "effect/Effect";
		4	import * as FileSystem from "effect/FileSystem";
4	import * as Layer from "effect/Layer";	5	import * as Layer from "effect/Layer";
5	import * as Option from "effect/Option";	6	import * as Option from "effect/Option";
6	import * as Path from "effect/Path";	7	import * as Path from "effect/Path";
@@ -22,6 +23,7 @@ import { writeServiceState } from "../serviceLauncher.ts"; · 15 unmodified lines
22	import {	23	import {
23	SERVICE_LAUNCHER_PROTOCOL,	24	SERVICE_LAUNCHER_PROTOCOL,
24	SERVICE_STATE_FILE,	25	SERVICE_STATE_FILE,
		26	SERVICE_STOP_MARKER_FILE,
25	type ServiceUpdateRecord,	27	type ServiceUpdateRecord,
26	} from "./serviceProtocol.ts";	28	} from "./serviceProtocol.ts";
27	import * as ServerEnvironment from "../environment/ServerEnvironment.ts";	29	import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
@@ -451,6 +453,35 @@ describe("releaseManagedTunnelOnShutdown", () => { · 423 unmodified lines
451	}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));	453	}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
452	});	454	});
453		455
		456	it.effect("still releases a pending update when the launcher is stopping", () => {
		457	// `t3 service uninstall` or `systemctl stop` during the pending window:
		458	// the launcher writes its stop marker before signalling the child, so no
		459	// replacement server is coming and the tunnel must not be kept.
		460	const { store, values } = makeMemorySecretStore(managedLinkSecrets);
		461	const applyConfigCalls: Array<unknown> = [];
		462	const requests: Array<HttpClientRequest.HttpClientRequest> = [];
		463
		464	return Effect.gen(function* () {
		465	yield* writeLauncherState({
		466	id: "update-1",
		467	fromVersion: "0.0.30",
		468	targetVersion: "0.0.31",
		469	dbPath: "/tmp/state.sqlite",
		470	status: "pending",
		471	});
		472	const fs = yield* FileSystem.FileSystem;
		473	const path = yield* Path.Path;
		474	const config = yield* ServerConfigModule.ServerConfig;
		475	yield* fs.writeFileString(path.join(config.baseDir, "runtime", SERVICE_STOP_MARKER_FILE), "");
		476
		477	const released = yield* releaseManagedTunnelOnShutdown();
		478
		479	expect(released).toBe(true);
		480	expect(requests).toHaveLength(1);
		481	expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false);
		482	}).pipe(provideReleaseHarness({ store, applyConfigCalls, requests }));
		483	});
		484
454	it.effect("still releases when the recorded update already settled", () => {	485	it.effect("still releases when the recorded update already settled", () => {
455	const { store, values } = makeMemorySecretStore(managedLinkSecrets);	486	const { store, values } = makeMemorySecretStore(managedLinkSecrets);
456	const applyConfigCalls: Array<unknown> = [];	487	const applyConfigCalls: Array<unknown> = [];

apps/server/src/cloud/http.ts+22−5

 import * as ServerConfig from "../config.ts";
 import * as ServerEnvironment from "../environment/ServerEnvironment.ts";
 import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts";
-import { SERVICE_STATE_FILE, serviceStateHasPendingUpdate } from "./serviceProtocol.ts";
+import {
+  SERVICE_STATE_FILE,
+  SERVICE_STOP_MARKER_FILE,
+  serviceStateHasPendingUpdate,
+} from "./serviceProtocol.ts";
 import {
   CLOUD_ENDPOINT_RUNTIME_CONFIG,
   CLOUD_LINKED_USER_ID,
 // straight from disk: the launcher owns that file, and this runs while the
 // server is tearing down. Unreadable or unparseable state means no pending
 // update — the release then proceeds as before.
-const pendingServiceUpdateExists = Effect.gen(function* () {
+//
+// A pending update alone is not proof a replacement server is coming: an
+// explicit launcher stop (`t3 service uninstall`, `systemctl stop`) during
+// the pending window also tears this server down, with nothing starting
+// afterwards. The launcher marks that case with a stop-marker file just
+// before it signals the child, so pending + no marker is the update handoff.
+const pendingUpdateHandoffExists = Effect.gen(function* () {
   const config = yield* ServerConfig.ServerConfig;
   const fs = yield* FileSystem.FileSystem;
   const path = yield* Path.Path;
+  const runtimeDir = path.join(config.baseDir, "runtime");
   const stateText = yield* fs
-    .readFileString(path.join(config.baseDir, "runtime", SERVICE_STATE_FILE))
+    .readFileString(path.join(runtimeDir, SERVICE_STATE_FILE))
     .pipe(Effect.option);
-  return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value);
+  if (Option.isNone(stateText) || !serviceStateHasPendingUpdate(stateText.value)) {
+    return false;
+  }
+  const stopping = yield* fs
+    .exists(path.join(runtimeDir, SERVICE_STOP_MARKER_FILE))
+    .pipe(Effect.orElseSucceed(() => false));
+  return !stopping;
 });
 // Cloudflare bills per provisioned tunnel, so an environment that goes offline
   // boot respawns the connector from the stored config and is reachable as
   // soon as it connects, and the reconcile confirms the still-live tunnel
   // without replacing it.
-  if (yield* pendingServiceUpdateExists) {
+  if (yield* pendingUpdateHandoffExists) {
     yield* Effect.logInfo("Keeping the managed tunnel across the update restart");
     return false;
   }

apps/server/src/cloud/serviceProtocol.ts+4−0

 export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT";
 export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs";
 export const SERVICE_STATE_FILE = "service-state.json";
+/** Written by the launcher just before an explicit stop kills its child, so
+    the child can tell "the service is going away" from "the launcher is about
+    to start my replacement" while a pending update is recorded. */
+export const SERVICE_STOP_MARKER_FILE = ".service-stopping";
 export interface PendingServiceUpdate {
   readonly id: string;

apps/server/src/serviceLauncher.test.ts+26−0

@@ -1,3 +1,6 @@
		1	// @effect-diagnostics-next-line nodeBuiltinImport:off - sync marker poll outside the test clock.
		2	import * as NodeFS from "node:fs";
		3
1	import * as NodeServices from "@effect/platform-node/NodeServices";	4	import * as NodeServices from "@effect/platform-node/NodeServices";
2	import { assert, it } from "@effect/vitest";	5	import { assert, it } from "@effect/vitest";
3	import * as Effect from "effect/Effect";	6	import * as Effect from "effect/Effect";
@@ -10,6 +13,7 @@ import { · 6 unmodified lines
10	decodeServiceState,	13	decodeServiceState,
11	isExactServiceVersion,	14	isExactServiceVersion,
12	SERVICE_LAUNCHER_PROTOCOL,	15	SERVICE_LAUNCHER_PROTOCOL,
		16	SERVICE_STOP_MARKER_FILE,
13	} from "./cloud/serviceProtocol.ts";	17	} from "./cloud/serviceProtocol.ts";
14		18
15	it("accepts only exact semantic versions", () => {	19	it("accepts only exact semantic versions", () => {
@@ -113,6 +117,28 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { · 97 unmodified lines
113	const running = launcher.run();	117	const running = launcher.run();
114	yield* Effect.promise(() => launcher.stop("SIGTERM"));	118	yield* Effect.promise(() => launcher.stop("SIGTERM"));
115	yield* Effect.promise(() => running);	119	yield* Effect.promise(() => running);
		120	// An explicit stop leaves the marker that tells a child shutting down
		121	// mid-update that no replacement server is coming.
		122	assert.isTrue(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE)));
		123
		124	// The next launcher start clears the stale marker so a later update
		125	// handoff keeps its tunnel again. Recovery runs before any stop
		126	// transition, so polling until the marker disappears observes it; the
		127	// poll uses real timers because it.effect runs under the test clock.
		128	const restarted = new Launcher(
		129	root,
		130	yield* Effect.promise(() => readServiceState(statePath)),
		131	);
		132	const runningAgain = restarted.run();
		133	const markerPath = path.join(root, "runtime", SERVICE_STOP_MARKER_FILE);
		134	yield* Effect.promise(async () => {
		135	while (NodeFS.existsSync(markerPath)) {
		136	// @effect-diagnostics-next-line globalTimers:off - real delay under the test clock.
		137	await new Promise((resolve) => setTimeout(resolve, 25));
		138	}
		139	});
		140	yield* Effect.promise(() => restarted.stop("SIGTERM"));
		141	yield* Effect.promise(() => runningAgain);
116	}),	142	}),
117	);	143	);
118		144

apps/server/src/serviceLauncher.ts+17−0

   SERVICE_LAUNCHER_CONTEXT_ENV,
   SERVICE_LAUNCHER_PROTOCOL,
   SERVICE_STATE_FILE,
+  SERVICE_STOP_MARKER_FILE,
 } from "./cloud/serviceProtocol.ts";
 const HANDOFF_DELAY_MS = 2_000;
   }
 }
+const stopMarkerPath = (baseDir: string) =>
+  NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE);
 export class Launcher {
   readonly #baseDir: string;
   readonly #statePath: string;
     this.#stopping = true;
     this.#clearTimer();
     this.#enqueue(async () => {
+      // An explicit stop is the service going away, not an update handoff.
+      // The marker lets the child's shutdown release the managed tunnel even
+      // while an update is recorded as pending; the next launcher start
+      // clears it. Written before the child is signalled so the child sees
+      // it during teardown. Best-effort: if the write fails the child errs
+      // toward keeping the tunnel, and the next link or unlink reconciles it.
+      await NodeFSP.writeFile(stopMarkerPath(this.#baseDir), "", { mode: 0o600 }).catch(
+        () => undefined,
+      );
       const child = this.#child?.process;
       this.#child = null;
       if (child !== undefined) await terminateChild(child, signal);
   }
   async #recover(): Promise<void> {
+    // A fresh launcher means servers are running again: any stop marker from
+    // a previous explicit stop is stale and must not make a future update
+    // handoff release its tunnel.
+    await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined);
     const update = this.#state.update;
     if (update?.status !== "pending") {
       if (update !== undefined) {