A pending update in the launcher state is not proof a replacement server is coming: `t3 service uninstall` or `systemctl stop` during the pending window also tears the server down, permanently. The launcher now writes a stop marker before signalling its child on an explicit stop and clears it on the next start; the shutdown tunnel release keeps the tunnel only for pending updates without the marker.
Unified · Split
| @@ -1,6 +1,7 @@ | ||
| 1 | 1 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 2 | 2 | import { describe, expect, it } from "@effect/vitest"; |
| 3 | 3 | import * as Effect from "effect/Effect"; |
| 4 | +import * as FileSystem from "effect/FileSystem"; | |
| 4 | 5 | import * as Layer from "effect/Layer"; |
| 5 | 6 | import * as Option from "effect/Option"; |
| 6 | 7 | import * as Path from "effect/Path"; |
| @@ -22,6 +23,7 @@ import { writeServiceState } from "../serviceLauncher.ts"; · 15 unmodified lines | ||
| 22 | 23 | import { |
| 23 | 24 | SERVICE_LAUNCHER_PROTOCOL, |
| 24 | 25 | SERVICE_STATE_FILE, |
| 26 | + SERVICE_STOP_MARKER_FILE, | |
| 25 | 27 | type ServiceUpdateRecord, |
| 26 | 28 | } from "./serviceProtocol.ts"; |
| 27 | 29 | import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; |
| @@ -451,6 +453,35 @@ describe("releaseManagedTunnelOnShutdown", () => { · 423 unmodified lines | ||
| 451 | 453 | }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); |
| 452 | 454 | }); |
| 453 | 455 | |
| 456 | + it.effect("still releases a pending update when the launcher is stopping", () => { | |
| 457 | + // `t3 service uninstall` or `systemctl stop` during the pending window: | |
| 458 | + // the launcher writes its stop marker before signalling the child, so no | |
| 459 | + // replacement server is coming and the tunnel must not be kept. | |
| 460 | + const { store, values } = makeMemorySecretStore(managedLinkSecrets); | |
| 461 | + const applyConfigCalls: Array<unknown> = []; | |
| 462 | + const requests: Array<HttpClientRequest.HttpClientRequest> = []; | |
| 463 | + | |
| 464 | + return Effect.gen(function* () { | |
| 465 | + yield* writeLauncherState({ | |
| 466 | + id: "update-1", | |
| 467 | + fromVersion: "0.0.30", | |
| 468 | + targetVersion: "0.0.31", | |
| 469 | + dbPath: "/tmp/state.sqlite", | |
| 470 | + status: "pending", | |
| 471 | + }); | |
| 472 | + const fs = yield* FileSystem.FileSystem; | |
| 473 | + const path = yield* Path.Path; | |
| 474 | + const config = yield* ServerConfigModule.ServerConfig; | |
| 475 | + yield* fs.writeFileString(path.join(config.baseDir, "runtime", SERVICE_STOP_MARKER_FILE), ""); | |
| 476 | + | |
| 477 | + const released = yield* releaseManagedTunnelOnShutdown(); | |
| 478 | + | |
| 479 | + expect(released).toBe(true); | |
| 480 | + expect(requests).toHaveLength(1); | |
| 481 | + expect(values.has(CLOUD_ENDPOINT_RUNTIME_CONFIG)).toBe(false); | |
| 482 | + }).pipe(provideReleaseHarness({ store, applyConfigCalls, requests })); | |
| 483 | + }); | |
| 484 | + | |
| 454 | 485 | it.effect("still releases when the recorded update already settled", () => { |
| 455 | 486 | const { store, values } = makeMemorySecretStore(managedLinkSecrets); |
| 456 | 487 | const applyConfigCalls: Array<unknown> = []; |
| @@ -61,7 +61,11 @@ import { requireEnvironmentScope } from "../auth/http.ts"; · 60 unmodified lines | ||
| 61 | 61 | import * as ServerConfig from "../config.ts"; |
| 62 | 62 | import * as ServerEnvironment from "../environment/ServerEnvironment.ts"; |
| 63 | 63 | import * as ManagedEndpointRuntime from "./ManagedEndpointRuntime.ts"; |
| 64 | −import { SERVICE_STATE_FILE, serviceStateHasPendingUpdate } from "./serviceProtocol.ts"; | |
| 64 | +import { | |
| 65 | + SERVICE_STATE_FILE, | |
| 66 | + SERVICE_STOP_MARKER_FILE, | |
| 67 | + serviceStateHasPendingUpdate, | |
| 68 | +} from "./serviceProtocol.ts"; | |
| 65 | 69 | import { |
| 66 | 70 | CLOUD_ENDPOINT_RUNTIME_CONFIG, |
| 67 | 71 | CLOUD_LINKED_USER_ID, |
| @@ -635,14 +639,27 @@ export const reconcileDesiredCloudLink = Effect.fn("environment.cloud.reconcileD · 567 unmodified lines | ||
| 635 | 639 | // straight from disk: the launcher owns that file, and this runs while the |
| 636 | 640 | // server is tearing down. Unreadable or unparseable state means no pending |
| 637 | 641 | // update — the release then proceeds as before. |
| 638 | −const pendingServiceUpdateExists = Effect.gen(function* () { | |
| 642 | +// | |
| 643 | +// A pending update alone is not proof a replacement server is coming: an | |
| 644 | +// explicit launcher stop (`t3 service uninstall`, `systemctl stop`) during | |
| 645 | +// the pending window also tears this server down, with nothing starting | |
| 646 | +// afterwards. The launcher marks that case with a stop-marker file just | |
| 647 | +// before it signals the child, so pending + no marker is the update handoff. | |
| 648 | +const pendingUpdateHandoffExists = Effect.gen(function* () { | |
| 639 | 649 | const config = yield* ServerConfig.ServerConfig; |
| 640 | 650 | const fs = yield* FileSystem.FileSystem; |
| 641 | 651 | const path = yield* Path.Path; |
| 652 | + const runtimeDir = path.join(config.baseDir, "runtime"); | |
| 642 | 653 | const stateText = yield* fs |
| 643 | − .readFileString(path.join(config.baseDir, "runtime", SERVICE_STATE_FILE)) | |
| 654 | + .readFileString(path.join(runtimeDir, SERVICE_STATE_FILE)) | |
| 644 | 655 | .pipe(Effect.option); |
| 645 | − return Option.isSome(stateText) && serviceStateHasPendingUpdate(stateText.value); | |
| 656 | + if (Option.isNone(stateText) || !serviceStateHasPendingUpdate(stateText.value)) { | |
| 657 | + return false; | |
| 658 | + } | |
| 659 | + const stopping = yield* fs | |
| 660 | + .exists(path.join(runtimeDir, SERVICE_STOP_MARKER_FILE)) | |
| 661 | + .pipe(Effect.orElseSucceed(() => false)); | |
| 662 | + return !stopping; | |
| 646 | 663 | }); |
| 647 | 664 | |
| 648 | 665 | // Cloudflare bills per provisioned tunnel, so an environment that goes offline |
| @@ -676,7 +693,7 @@ export const releaseManagedTunnelOnShutdown = Effect.fn( · 27 unmodified lines | ||
| 676 | 693 | // boot respawns the connector from the stored config and is reachable as |
| 677 | 694 | // soon as it connects, and the reconcile confirms the still-live tunnel |
| 678 | 695 | // without replacing it. |
| 679 | − if (yield* pendingServiceUpdateExists) { | |
| 696 | + if (yield* pendingUpdateHandoffExists) { | |
| 680 | 697 | yield* Effect.logInfo("Keeping the managed tunnel across the update restart"); |
| 681 | 698 | return false; |
| 682 | 699 | } |
| @@ -5,6 +5,10 @@ export const SERVICE_LAUNCHER_PROTOCOL = 2 as const; · 4 unmodified lines | ||
| 5 | 5 | export const SERVICE_LAUNCHER_CONTEXT_ENV = "T3_SERVICE_LAUNCHER_CONTEXT"; |
| 6 | 6 | export const SERVICE_LAUNCHER_FILE = "service-launcher.mjs"; |
| 7 | 7 | export const SERVICE_STATE_FILE = "service-state.json"; |
| 8 | +/** Written by the launcher just before an explicit stop kills its child, so | |
| 9 | + the child can tell "the service is going away" from "the launcher is about | |
| 10 | + to start my replacement" while a pending update is recorded. */ | |
| 11 | +export const SERVICE_STOP_MARKER_FILE = ".service-stopping"; | |
| 8 | 12 | |
| 9 | 13 | export interface PendingServiceUpdate { |
| 10 | 14 | readonly id: string; |
| @@ -1,3 +1,6 @@ | ||
| 1 | +// @effect-diagnostics-next-line nodeBuiltinImport:off - sync marker poll outside the test clock. | |
| 2 | +import * as NodeFS from "node:fs"; | |
| 3 | + | |
| 1 | 4 | import * as NodeServices from "@effect/platform-node/NodeServices"; |
| 2 | 5 | import { assert, it } from "@effect/vitest"; |
| 3 | 6 | import * as Effect from "effect/Effect"; |
| @@ -10,6 +13,7 @@ import { · 6 unmodified lines | ||
| 10 | 13 | decodeServiceState, |
| 11 | 14 | isExactServiceVersion, |
| 12 | 15 | SERVICE_LAUNCHER_PROTOCOL, |
| 16 | + SERVICE_STOP_MARKER_FILE, | |
| 13 | 17 | } from "./cloud/serviceProtocol.ts"; |
| 14 | 18 | |
| 15 | 19 | it("accepts only exact semantic versions", () => { |
| @@ -113,6 +117,28 @@ it.layer(NodeServices.layer)("service state persistence", (it) => { · 97 unmodified lines | ||
| 113 | 117 | const running = launcher.run(); |
| 114 | 118 | yield* Effect.promise(() => launcher.stop("SIGTERM")); |
| 115 | 119 | yield* Effect.promise(() => running); |
| 120 | + // An explicit stop leaves the marker that tells a child shutting down | |
| 121 | + // mid-update that no replacement server is coming. | |
| 122 | + assert.isTrue(yield* fs.exists(path.join(root, "runtime", SERVICE_STOP_MARKER_FILE))); | |
| 123 | + | |
| 124 | + // The next launcher start clears the stale marker so a later update | |
| 125 | + // handoff keeps its tunnel again. Recovery runs before any stop | |
| 126 | + // transition, so polling until the marker disappears observes it; the | |
| 127 | + // poll uses real timers because it.effect runs under the test clock. | |
| 128 | + const restarted = new Launcher( | |
| 129 | + root, | |
| 130 | + yield* Effect.promise(() => readServiceState(statePath)), | |
| 131 | + ); | |
| 132 | + const runningAgain = restarted.run(); | |
| 133 | + const markerPath = path.join(root, "runtime", SERVICE_STOP_MARKER_FILE); | |
| 134 | + yield* Effect.promise(async () => { | |
| 135 | + while (NodeFS.existsSync(markerPath)) { | |
| 136 | + // @effect-diagnostics-next-line globalTimers:off - real delay under the test clock. | |
| 137 | + await new Promise((resolve) => setTimeout(resolve, 25)); | |
| 138 | + } | |
| 139 | + }); | |
| 140 | + yield* Effect.promise(() => restarted.stop("SIGTERM")); | |
| 141 | + yield* Effect.promise(() => runningAgain); | |
| 116 | 142 | }), |
| 117 | 143 | ); |
| 118 | 144 | |
| @@ -24,6 +24,7 @@ import { · 23 unmodified lines | ||
| 24 | 24 | SERVICE_LAUNCHER_CONTEXT_ENV, |
| 25 | 25 | SERVICE_LAUNCHER_PROTOCOL, |
| 26 | 26 | SERVICE_STATE_FILE, |
| 27 | + SERVICE_STOP_MARKER_FILE, | |
| 27 | 28 | } from "./cloud/serviceProtocol.ts"; |
| 28 | 29 | |
| 29 | 30 | const HANDOFF_DELAY_MS = 2_000; |
| @@ -257,6 +258,9 @@ async function terminateChild( · 227 unmodified lines | ||
| 257 | 258 | } |
| 258 | 259 | } |
| 259 | 260 | |
| 261 | +const stopMarkerPath = (baseDir: string) => | |
| 262 | + NodePath.join(baseDir, "runtime", SERVICE_STOP_MARKER_FILE); | |
| 263 | + | |
| 260 | 264 | export class Launcher { |
| 261 | 265 | readonly #baseDir: string; |
| 262 | 266 | readonly #statePath: string; |
| @@ -315,6 +319,15 @@ export class Launcher { · 52 unmodified lines | ||
| 315 | 319 | this.#stopping = true; |
| 316 | 320 | this.#clearTimer(); |
| 317 | 321 | this.#enqueue(async () => { |
| 322 | + // An explicit stop is the service going away, not an update handoff. | |
| 323 | + // The marker lets the child's shutdown release the managed tunnel even | |
| 324 | + // while an update is recorded as pending; the next launcher start | |
| 325 | + // clears it. Written before the child is signalled so the child sees | |
| 326 | + // it during teardown. Best-effort: if the write fails the child errs | |
| 327 | + // toward keeping the tunnel, and the next link or unlink reconciles it. | |
| 328 | + await NodeFSP.writeFile(stopMarkerPath(this.#baseDir), "", { mode: 0o600 }).catch( | |
| 329 | + () => undefined, | |
| 330 | + ); | |
| 318 | 331 | const child = this.#child?.process; |
| 319 | 332 | this.#child = null; |
| 320 | 333 | if (child !== undefined) await terminateChild(child, signal); |
| @@ -330,6 +343,10 @@ export class Launcher { · 9 unmodified lines | ||
| 330 | 343 | } |
| 331 | 344 | |
| 332 | 345 | async #recover(): Promise<void> { |
| 346 | + // A fresh launcher means servers are running again: any stop marker from | |
| 347 | + // a previous explicit stop is stale and must not make a future update | |
| 348 | + // handoff release its tunnel. | |
| 349 | + await NodeFSP.rm(stopMarkerPath(this.#baseDir), { force: true }).catch(() => undefined); | |
| 333 | 350 | const update = this.#state.update; |
| 334 | 351 | if (update?.status !== "pending") { |
| 335 | 352 | if (update !== undefined) { |