History

Make relay install lock more idiomatic Effect

Cursor Agent committed 5 months ago

Commit 5feb87d1 parent 3ea6adf2 files · +197 −54Browse files

Unified · Split

2 files
  1. packages/shared/src/relayClient.test.ts +118 −17
  2. packages/shared/src/relayClient.ts +79 −37
packages/shared/src/relayClient.test.ts+118−17
@@ -1,13 +1,17 @@
1import { sha256 } from "@noble/hashes/sha2";1import { sha256 } from "@noble/hashes/sha2";
2import * as NodeServices from "@effect/platform-node/NodeServices";2import * as NodeServices from "@effect/platform-node/NodeServices";
3import { describe, expect, it } from "@effect/vitest";3import { assert, describe, it } from "@effect/vitest";
4import * as ConfigProvider from "effect/ConfigProvider";4import * as ConfigProvider from "effect/ConfigProvider";
5import * as Duration from "effect/Duration";
5import * as Effect from "effect/Effect";6import * as Effect from "effect/Effect";
6import * as Encoding from "effect/Encoding";7import * as Encoding from "effect/Encoding";
7import * as FileSystem from "effect/FileSystem";8import * as FileSystem from "effect/FileSystem";
9import * as Fiber from "effect/Fiber";
8import * as Layer from "effect/Layer";10import * as Layer from "effect/Layer";
11import * as Path from "effect/Path";
9import * as Sink from "effect/Sink";12import * as Sink from "effect/Sink";
10import * as Stream from "effect/Stream";13import * as Stream from "effect/Stream";
14import * as TestClock from "effect/testing/TestClock";
11import { HttpClient, HttpClientResponse } from "effect/unstable/http";15import { HttpClient, HttpClientResponse } from "effect/unstable/http";
12import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";16import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
13 17
@@ -47,15 +51,13 @@ const makeHttpClientLayer = (bytes: Uint8Array) => · 33 unmodified lines
47 );51 );
48 52
49const makeSpawnerLayer = (commands: Array<string>) =>53const makeSpawnerLayer = (commands: Array<string>) =>
50 Layer.succeed(54 Layer.mock(ChildProcessSpawner.ChildProcessSpawner, {
51 ChildProcessSpawner.ChildProcessSpawner,55 spawn: (command) =>
52 ChildProcessSpawner.make((command) =>
53 Effect.sync(() => {56 Effect.sync(() => {
54 commands.push(ChildProcess.isStandardCommand(command) ? command.command : "piped-command");57 commands.push(ChildProcess.isStandardCommand(command) ? command.command : "piped-command");
55 return makeHandle();58 return makeHandle();
56 }),59 }),
57 ),60 });
58 );
59 61
60describe("RelayClient", () => {62describe("RelayClient", () => {
61 it.effect("resolves explicit overrides before managed and PATH executables", () =>63 it.effect("resolves explicit overrides before managed and PATH executables", () =>
@@ -80,7 +82,7 @@ describe("RelayClient", () => { · 18 unmodified lines
80 }),82 }),
81 });83 });
82 84
83 expect(yield* manager.resolve).toEqual({85 assert.deepStrictEqual(yield* manager.resolve, {
84 status: "available",86 status: "available",
85 executablePath: overridePath,87 executablePath: overridePath,
86 source: "override",88 source: "override",
@@ -130,16 +132,17 @@ describe("RelayClient", () => { · 43 unmodified lines
130 platform: "linux",132 platform: "linux",
131 arch: "x64",133 arch: "x64",
132 });134 });
133 expect(installed).toEqual({135 assert.deepStrictEqual(installed, {
134 status: "available",136 status: "available",
135 executablePath: managedPath,137 executablePath: managedPath,
136 source: "managed",138 source: "managed",
137 version: CLOUDFLARED_VERSION,139 version: CLOUDFLARED_VERSION,
138 });140 });
139 expect(new TextDecoder().decode(yield* fileSystem.readFile(managedPath))).toBe(141 assert.equal(
142 new TextDecoder().decode(yield* fileSystem.readFile(managedPath)),
140 "test-cloudflared-binary",143 "test-cloudflared-binary",
141 );144 );
142 expect(progress).toEqual([145 assert.deepStrictEqual(progress, [
143 "checking",146 "checking",
144 "waiting_for_lock",147 "waiting_for_lock",
145 "downloading",148 "downloading",
@@ -148,7 +151,7 @@ describe("RelayClient", () => { · 2 unmodified lines
148 "validating",151 "validating",
149 "activating",152 "activating",
150 ]);153 ]);
151 expect(yield* manager.resolve).toEqual(installed);154 assert.deepStrictEqual(yield* manager.resolve, installed);
152 }).pipe(155 }).pipe(
153 Effect.scoped,156 Effect.scoped,
154 Effect.provide(157 Effect.provide(
@@ -180,8 +183,8 @@ describe("RelayClient", () => { · 25 unmodified lines
180 });183 });
181 184
182 const error = yield* manager.install.pipe(Effect.flip);185 const error = yield* manager.install.pipe(Effect.flip);
183 expect(error).toBeInstanceOf(RelayClientInstallError);186 assert.ok(error instanceof RelayClientInstallError);
184 expect(error.reason).toBe("invalid_checksum");187 assert.equal(error.reason, "invalid_checksum");
185 }).pipe(188 }).pipe(
186 Effect.scoped,189 Effect.scoped,
187 Effect.provide(190 Effect.provide(
@@ -217,8 +220,8 @@ describe("RelayClient", () => { · 29 unmodified lines
217 const [first, second] = yield* Effect.all([manager.install, manager.install], {220 const [first, second] = yield* Effect.all([manager.install, manager.install], {
218 concurrency: "unbounded",221 concurrency: "unbounded",
219 });222 });
220 expect(second).toEqual(first);223 assert.deepStrictEqual(second, first);
221 expect(commands).toHaveLength(1);224 assert.equal(commands.length, 1);
222 }).pipe(225 }).pipe(
223 Effect.scoped,226 Effect.scoped,
224 Effect.provide(227 Effect.provide(
@@ -243,7 +246,7 @@ describe("RelayClient", () => { · 18 unmodified lines
243 configProvider: () => ConfigProvider.fromEnv({ env: { PATH: path } }),246 configProvider: () => ConfigProvider.fromEnv({ env: { PATH: path } }),
244 });247 });
245 248
246 expect(yield* manager.resolve).toEqual({249 assert.deepStrictEqual(yield* manager.resolve, {
247 status: "missing",250 status: "missing",
248 version: CLOUDFLARED_VERSION,251 version: CLOUDFLARED_VERSION,
249 });252 });
@@ -253,7 +256,7 @@ describe("RelayClient", () => { · 3 unmodified lines
253 yield* fileSystem.chmod(executablePath, 0o755);256 yield* fileSystem.chmod(executablePath, 0o755);
254 path = binDir;257 path = binDir;
255 258
256 expect(yield* manager.resolve).toEqual({259 assert.deepStrictEqual(yield* manager.resolve, {
257 status: "available",260 status: "available",
258 executablePath,261 executablePath,
259 source: "path",262 source: "path",
@@ -270,4 +273,102 @@ describe("RelayClient", () => { · 10 unmodified lines
270 ),273 ),
271 ),274 ),
272 );275 );
276
277 it.effect("removes stale install locks before installing", () => {
278 const commands: Array<string> = [];
279 const bytes = new TextEncoder().encode("test-cloudflared-binary");
280 return Effect.gen(function* () {
281 const fileSystem = yield* FileSystem.FileSystem;
282 const path = yield* Path.Path;
283 const baseDir = yield* fileSystem.makeTempDirectoryScoped({
284 prefix: "t3-cloudflared-test-",
285 });
286 const managedPath = resolveManagedCloudflaredPath({
287 baseDir,
288 platform: "linux",
289 arch: "x64",
290 });
291 const lockPath = `${managedPath}.lock`;
292 yield* fileSystem.makeDirectory(path.dirname(lockPath), { recursive: true });
293 yield* fileSystem.writeFileString(lockPath, "stale");
294 yield* fileSystem.utimes(lockPath, 0, 0);
295 yield* TestClock.adjust(Duration.minutes(6));
296
297 const manager = yield* makeCloudflaredRelayClient({
298 baseDir,
299 platform: "linux",
300 arch: "x64",
301 releaseAsset: {
302 url: "https://example.test/cloudflared",
303 sha256: Encoding.encodeHex(sha256(bytes)),
304 archive: "binary",
305 },
306 configProvider: emptyConfigProvider,
307 });
308
309 const installed = yield* manager.install;
310
311 assert.deepStrictEqual(installed, {
312 status: "available",
313 executablePath: managedPath,
314 source: "managed",
315 version: CLOUDFLARED_VERSION,
316 });
317 assert.equal(yield* fileSystem.exists(lockPath), false);
318 assert.equal(commands.length, 1);
319 }).pipe(
320 Effect.scoped,
321 Effect.provide(
322 Layer.mergeAll(NodeServices.layer, makeHttpClientLayer(bytes), makeSpawnerLayer(commands)),
323 ),
324 );
325 });
326
327 it.effect("times out install lock waits with the test clock", () => {
328 const bytes = new TextEncoder().encode("test-cloudflared-binary");
329 return Effect.gen(function* () {
330 const fileSystem = yield* FileSystem.FileSystem;
331 const path = yield* Path.Path;
332 const baseDir = yield* fileSystem.makeTempDirectoryScoped({
333 prefix: "t3-cloudflared-test-",
334 });
335 const managedPath = resolveManagedCloudflaredPath({
336 baseDir,
337 platform: "linux",
338 arch: "x64",
339 });
340 const lockPath = `${managedPath}.lock`;
341 yield* fileSystem.makeDirectory(path.dirname(lockPath), { recursive: true });
342 yield* fileSystem.writeFileString(lockPath, "locked");
343
344 const manager = yield* makeCloudflaredRelayClient({
345 baseDir,
346 platform: "linux",
347 arch: "x64",
348 releaseAsset: {
349 url: "https://example.test/cloudflared",
350 sha256: Encoding.encodeHex(sha256(bytes)),
351 archive: "binary",
352 },
353 configProvider: emptyConfigProvider,
354 });
355
356 const install = yield* manager.install.pipe(Effect.flip, Effect.fork);
357 yield* TestClock.adjust(Duration.seconds(10));
358 const error = yield* Fiber.join(install);
359
360 assert.ok(error instanceof RelayClientInstallError);
361 assert.equal(error.reason, "install_locked");
362 assert.equal(yield* fileSystem.exists(lockPath), true);
363 }).pipe(
364 Effect.scoped,
365 Effect.provide(
366 Layer.mergeAll(
367 NodeServices.layer,
368 makeHttpClientLayer(bytes),
369 makeSpawnerLayer([]),
370 ),
371 ),
372 );
373 });
273});374});
packages/shared/src/relayClient.ts+79−37
@@ -1,4 +1,3 @@
1import * as Clock from "effect/Clock";
2import type {1import type {
3 RelayClientInstallProgressEvent,2 RelayClientInstallProgressEvent,
4 RelayClientInstallProgressStage,3 RelayClientInstallProgressStage,
@@ -8,6 +7,8 @@ import * as ConfigProvider from "effect/ConfigProvider"; · 3 unmodified lines
8import * as Context from "effect/Context";7import * as Context from "effect/Context";
9import * as Crypto from "effect/Crypto";8import * as Crypto from "effect/Crypto";
10import * as Data from "effect/Data";9import * as Data from "effect/Data";
10import * as DateTime from "effect/DateTime";
11import * as Duration from "effect/Duration";
11import * as Effect from "effect/Effect";12import * as Effect from "effect/Effect";
12import * as Encoding from "effect/Encoding";13import * as Encoding from "effect/Encoding";
13import * as FileSystem from "effect/FileSystem";14import * as FileSystem from "effect/FileSystem";
@@ -15,6 +16,7 @@ import * as Layer from "effect/Layer"; · 1 unmodified line
15import * as Option from "effect/Option";16import * as Option from "effect/Option";
16import * as Path from "effect/Path";17import * as Path from "effect/Path";
17import * as PlatformError from "effect/PlatformError";18import * as PlatformError from "effect/PlatformError";
19import * as Schedule from "effect/Schedule";
18import * as Semaphore from "effect/Semaphore";20import * as Semaphore from "effect/Semaphore";
19import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";21import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http";
20import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";22import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process";
@@ -99,8 +101,18 @@ const CLOUDFLARED_RELEASE_ASSETS: Readonly< · 78 unmodified lines
99};101};
100 102
101const INSTALL_LOCK_RETRY_COUNT = 100;103const INSTALL_LOCK_RETRY_COUNT = 100;
102const INSTALL_LOCK_RETRY_DELAY = "100 millis";104const INSTALL_LOCK_RETRY_DELAY = Duration.millis(100);
103const INSTALL_LOCK_STALE_MS = 5 * 60 * 1_000;105const INSTALL_LOCK_TIMEOUT = Duration.times(INSTALL_LOCK_RETRY_DELAY, INSTALL_LOCK_RETRY_COUNT);
106const INSTALL_LOCK_STALE_AGE = Duration.minutes(5);
107
108class RelayClientInstallLockBusy extends Data.TaggedError("RelayClientInstallLockBusy")<{
109 readonly lockPath: string;
110}> {}
111
112const retryWhileInstallLockBusy = Schedule.spaced(INSTALL_LOCK_RETRY_DELAY).pipe(
113 Schedule.setInputType<RelayClientInstallLockBusy | PlatformError.PlatformError>(),
114 Schedule.while(({ input }) => input._tag === "RelayClientInstallLockBusy"),
115);
104 116
105const trimmedString = (name: string) =>117const trimmedString = (name: string) =>
106 Config.string(name).pipe(118 Config.string(name).pipe(
@@ -161,8 +173,8 @@ export function resolveManagedCloudflaredPath(input: { · 54 unmodified lines
161function resolveReleaseAsset(173function resolveReleaseAsset(
162 platform: NodeJS.Platform,174 platform: NodeJS.Platform,
163 arch: string,175 arch: string,
164): CloudflaredReleaseAsset | null {176): Option.Option<CloudflaredReleaseAsset> {
165 return CLOUDFLARED_RELEASE_ASSETS[`${platform}-${arch}`] ?? null;177 return Option.fromUndefinedOr(CLOUDFLARED_RELEASE_ASSETS[`${platform}-${arch}`]);
166}178}
167 179
168function isAlreadyExists(error: PlatformError.PlatformError): boolean {180function isAlreadyExists(error: PlatformError.PlatformError): boolean {
@@ -207,7 +219,9 @@ export const makeCloudflaredRelayClient = Effect.fn("cloudflared.make")(function · 38 unmodified lines
207 const installSemaphore = yield* Semaphore.make(1);219 const installSemaphore = yield* Semaphore.make(1);
208 const platform = options.platform ?? process.platform;220 const platform = options.platform ?? process.platform;
209 const arch = options.arch ?? process.arch;221 const arch = options.arch ?? process.arch;
210 const releaseAsset = options.releaseAsset ?? resolveReleaseAsset(platform, arch);222 const releaseAsset = Option.fromUndefinedOr(options.releaseAsset).pipe(
223 Option.orElse(() => resolveReleaseAsset(platform, arch)),
224 );
211 const loadCloudflaredConfig = Effect.suspend(() =>225 const loadCloudflaredConfig = Effect.suspend(() =>
212 CloudflaredConfig.pipe(226 CloudflaredConfig.pipe(
213 Effect.provideService(227 Effect.provideService(
@@ -236,15 +250,15 @@ export const makeCloudflaredRelayClient = Effect.fn("cloudflared.make")(function · 22 unmodified lines
236 const resolvePathExecutable = Effect.gen(function* () {250 const resolvePathExecutable = Effect.gen(function* () {
237 const config = yield* loadCloudflaredConfig;251 const config = yield* loadCloudflaredConfig;
238 const pathValue = Option.getOrUndefined(config.path);252 const pathValue = Option.getOrUndefined(config.path);
239 if (!pathValue) return null;253 if (!pathValue) return Option.none();
240 const delimiter = platform === "win32" ? ";" : ":";254 const delimiter = platform === "win32" ? ";" : ":";
241 for (const directory of pathValue.split(delimiter)) {255 for (const directory of pathValue.split(delimiter)) {
242 const trimmed = directory.trim().replace(/^"|"$/gu, "");256 const trimmed = directory.trim().replace(/^"|"$/gu, "");
243 if (trimmed.length === 0) continue;257 if (trimmed.length === 0) continue;
244 const candidate = path.join(trimmed, executableFileName(platform));258 const candidate = path.join(trimmed, executableFileName(platform));
245 if (yield* isExecutableFile(candidate)) return candidate;259 if (yield* isExecutableFile(candidate)) return Option.some(candidate);
246 }260 }
247 return null;261 return Option.none();
248 });262 });
249 263
250 const resolve: RelayClientShape["resolve"] = Effect.gen(function* () {264 const resolve: RelayClientShape["resolve"] = Effect.gen(function* () {
@@ -268,15 +282,15 @@ export const makeCloudflaredRelayClient = Effect.fn("cloudflared.make")(function · 17 unmodified lines
268 };282 };
269 }283 }
270 const pathExecutable = yield* resolvePathExecutable;284 const pathExecutable = yield* resolvePathExecutable;
271 if (pathExecutable) {285 if (Option.isSome(pathExecutable)) {
272 return {286 return {
273 status: "available",287 status: "available",
274 executablePath: pathExecutable,288 executablePath: pathExecutable.value,
275 source: "path",289 source: "path",
276 version: CLOUDFLARED_VERSION,290 version: CLOUDFLARED_VERSION,
277 };291 };
278 }292 }
279 return releaseAsset293 return Option.isSome(releaseAsset)
280 ? { status: "missing", version: CLOUDFLARED_VERSION }294 ? { status: "missing", version: CLOUDFLARED_VERSION }
281 : {295 : {
282 status: "unsupported",296 status: "unsupported",
@@ -351,31 +365,59 @@ export const makeCloudflaredRelayClient = Effect.fn("cloudflared.make")(function · 68 unmodified lines
351 return bytes;365 return bytes;
352 });366 });
353 367
368 const isInstallLockStale = Effect.fn("cloudflared.isInstallLockStale")(function* (
369 lockPath: string,
370 ) {
371 const lockInfo = yield* fileSystem.stat(lockPath).pipe(Effect.option);
372 const lockModifiedAt = Option.flatMap(lockInfo, (info) => info.mtime);
373 if (Option.isNone(lockModifiedAt)) return false;
374
375 const now = yield* DateTime.now;
376 const lockAge = DateTime.distance(DateTime.makeUnsafe(lockModifiedAt.value), now);
377 return Duration.isGreaterThan(lockAge, INSTALL_LOCK_STALE_AGE);
378 });
379
380 const attemptAcquireInstallLock = Effect.fn("cloudflared.attemptAcquireInstallLock")(function* (
381 lockPath: string,
382 ) {
383 const acquired = yield* fileSystem.writeFileString(lockPath, "", { flag: "wx" }).pipe(
384 Effect.as(true),
385 Effect.catch((error) => (isAlreadyExists(error) ? Effect.succeed(false) : Effect.fail(error))),
386 );
387 if (acquired) return;
388
389 if (yield* isInstallLockStale(lockPath)) {
390 yield* fileSystem.remove(lockPath, { force: true });
391 return yield* attemptAcquireInstallLock(lockPath);
392 }
393
394 return yield* new RelayClientInstallLockBusy({ lockPath });
395 });
396
354 const acquireInstallLock = Effect.fn("cloudflared.acquireInstallLock")(function* (397 const acquireInstallLock = Effect.fn("cloudflared.acquireInstallLock")(function* (
355 lockPath: string,398 lockPath: string,
356 ) {399 ) {
357 for (let attempt = 0; attempt < INSTALL_LOCK_RETRY_COUNT; attempt += 1) {400 return yield* attemptAcquireInstallLock(lockPath).pipe(
358 const acquired = yield* fileSystem.writeFileString(lockPath, "", { flag: "wx" }).pipe(401 Effect.retry(retryWhileInstallLockBusy),
359 Effect.as(true),402 Effect.timeoutOrElse({
360 Effect.catch((error) =>403 duration: INSTALL_LOCK_TIMEOUT,
361 isAlreadyExists(error) ? Effect.succeed(false) : Effect.fail(error),404 orElse: () =>
405 Effect.fail(
406 new RelayClientInstallError({
407 reason: "install_locked",
408 message: "Another relay client installation is still in progress.",
409 }),
410 ),
411 }),
412 Effect.catchTag("RelayClientInstallLockBusy", () =>
413 Effect.fail(
414 new RelayClientInstallError({
415 reason: "install_locked",
416 message: "Another relay client installation is still in progress.",
417 }),
362 ),418 ),
363 );419 ),
364 if (acquired) return;420 );
365
366 const now = yield* Clock.currentTimeMillis;
367 const lockInfo = yield* fileSystem.stat(lockPath).pipe(Effect.option);
368 const mtime = Option.flatMap(lockInfo, (info) => info.mtime);
369 if (Option.isSome(mtime) && now - mtime.value.getTime() > INSTALL_LOCK_STALE_MS) {
370 yield* fileSystem.remove(lockPath, { force: true });
371 continue;
372 }
373 yield* Effect.sleep(INSTALL_LOCK_RETRY_DELAY);
374 }
375 return yield* new RelayClientInstallError({
376 reason: "install_locked",
377 message: "Another relay client installation is still in progress.",
378 });
379 });421 });
380 422
381 const installUnlocked = Effect.fn("cloudflared.installUnlocked")(function* (423 const installUnlocked = Effect.fn("cloudflared.installUnlocked")(function* (
@@ -391,7 +433,7 @@ export const makeCloudflaredRelayClient = Effect.fn("cloudflared.make")(function · 9 unmodified lines
391 message: `${CLOUDFLARED_PATH_ENV_NAME} does not point to an executable file.`,433 message: `${CLOUDFLARED_PATH_ENV_NAME} does not point to an executable file.`,
392 });434 });
393 }435 }
394 if (!releaseAsset) {436 if (Option.isNone(releaseAsset)) {
395 return yield* new RelayClientInstallError({437 return yield* new RelayClientInstallError({
396 reason: "unsupported_platform",438 reason: "unsupported_platform",
397 message: `T3 Code does not provide a managed relay client binary for ${platform}-${arch}.`,439 message: `T3 Code does not provide a managed relay client binary for ${platform}-${arch}.`,
@@ -427,16 +469,16 @@ export const makeCloudflaredRelayClient = Effect.fn("cloudflared.make")(function · 29 unmodified lines
427 });469 });
428 const archivePath = path.join(470 const archivePath = path.join(
429 tempDirectory,471 tempDirectory,
430 releaseAsset.archive === "tgz" ? "cloudflared.tgz" : executableFileName(platform),472 releaseAsset.value.archive === "tgz" ? "cloudflared.tgz" : executableFileName(platform),
431 );473 );
432 const download = yield* downloadAsset(releaseAsset, report);474 const download = yield* downloadAsset(releaseAsset.value, report);
433 yield* report("installing");475 yield* report("installing");
434 yield* fileSystem476 yield* fileSystem
435 .writeFile(archivePath, download)477 .writeFile(archivePath, download)
436 .pipe(wrapInstallFailure("write_failed", "Could not write the relay client download."));478 .pipe(wrapInstallFailure("write_failed", "Could not write the relay client download."));
437 479
438 const executablePath = path.join(tempDirectory, executableFileName(platform));480 const executablePath = path.join(tempDirectory, executableFileName(platform));
439 if (releaseAsset.archive === "tgz") {481 if (releaseAsset.value.archive === "tgz") {
440 yield* runCommand("tar", ["-xzf", archivePath, "-C", tempDirectory]).pipe(482 yield* runCommand("tar", ["-xzf", archivePath, "-C", tempDirectory]).pipe(
441 wrapInstallFailure("write_failed", "Could not extract the relay client."),483 wrapInstallFailure("write_failed", "Could not extract the relay client."),
442 );484 );