History

perf(ci): speed up release builds and Windows packaging (#7975)

Theo Browne committed 2 months ago

Commit 25dcee01 parent f70eeee6 files · +251 −120Browse files

Showing the changes to scripts (2 of 6 files). Show all

Unified · Split

2 files
  1. scripts/build-desktop-artifact.test.ts +119 −1
  2. scripts/build-desktop-artifact.ts +50 −22
scripts/build-desktop-artifact.test.ts+119−1
@@ -40,6 +40,7 @@ import { · 39 unmodified lines
40 resolveDesktopUpdateChannel,40 resolveDesktopUpdateChannel,
41 resolveDesktopWebAssetBrand,41 resolveDesktopWebAssetBrand,
42 resolveResourceMonitorRustTargets,42 resolveResourceMonitorRustTargets,
43 resolveWindowsServerAsarIgnoreGlobs,
43 resourceMonitorExecutableName,44 resourceMonitorExecutableName,
44 resolveGitHubPublishConfig,45 resolveGitHubPublishConfig,
45 resolveMockUpdateServerPort,46 resolveMockUpdateServerPort,
@@ -47,6 +48,7 @@ import { · 1 unmodified line
47 resolvePackageManagerUserAgent,48 resolvePackageManagerUserAgent,
48 stageLinuxIconSize,49 stageLinuxIconSize,
49 stageDesktopDmgBackground,50 stageDesktopDmgBackground,
51 stageResourceMonitor,
50 STAGE_INSTALL_ARGS,52 STAGE_INSTALL_ARGS,
51 ancestorNodeModulesPaths,53 ancestorNodeModulesPaths,
52 copyDirectoryPreservingSymlinks,54 copyDirectoryPreservingSymlinks,
@@ -118,7 +120,7 @@ const makeWindowsPayloadFixture = Effect.fn("test.makeWindowsPayloadFixture")(fu · 65 unmodified lines
118 yield* fs.writeFileString(nativePath, "native-binary");120 yield* fs.writeFileString(nativePath, "native-binary");
119 121
120 const generatedAsarPath = path.join(tempDir, WINDOWS_SERVER_ASAR_RESOURCE);122 const generatedAsarPath = path.join(tempDir, WINDOWS_SERVER_ASAR_RESOURCE);
121 yield* packWindowsServerAsar({ sourceDir, asarPath: generatedAsarPath });123 yield* packWindowsServerAsar({ sourceDir, asarPath: generatedAsarPath, arch: "x64" });
122 124
123 const stageDistDir = path.join(tempDir, "dist");125 const stageDistDir = path.join(tempDir, "dist");
124 const packagedAppDir = path.join(stageDistDir, "win-unpacked");126 const packagedAppDir = path.join(stageDistDir, "win-unpacked");
@@ -488,6 +490,121 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { · 363 unmodified lines
488 }).pipe(Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env: {} })))),490 }).pipe(Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env: {} })))),
489 );491 );
490 492
493 it("excludes node-pty binaries for the other Windows architecture", () => {
494 assert.deepStrictEqual(resolveWindowsServerAsarIgnoreGlobs("x64"), [
495 ...WINDOWS_SERVER_ASAR_IGNORE_GLOBS,
496 "**/node_modules/node-pty/prebuilds/win32-arm64",
497 "**/node_modules/node-pty/prebuilds/win32-arm64/**",
498 "**/node_modules/node-pty/third_party/conpty/*/win10-arm64",
499 "**/node_modules/node-pty/third_party/conpty/*/win10-arm64/**",
500 ]);
501 assert.deepStrictEqual(resolveWindowsServerAsarIgnoreGlobs("arm64"), [
502 ...WINDOWS_SERVER_ASAR_IGNORE_GLOBS,
503 "**/node_modules/node-pty/prebuilds/win32-x64",
504 "**/node_modules/node-pty/prebuilds/win32-x64/**",
505 "**/node_modules/node-pty/third_party/conpty/*/win10-x64",
506 "**/node_modules/node-pty/third_party/conpty/*/win10-x64/**",
507 ]);
508 });
509
510 it.effect(
511 "keeps target and WSL native files while excluding the other Windows architecture",
512 () =>
513 Effect.scoped(
514 Effect.gen(function* () {
515 const fs = yield* FileSystem.FileSystem;
516 const path = yield* Path.Path;
517 const tempDir = yield* fs.makeTempDirectoryScoped({
518 prefix: "t3-windows-architecture-test-",
519 });
520 const sourceDir = path.join(tempDir, "server");
521 const nativeFiles = [
522 "node_modules/node-pty/prebuilds/win32-x64/conpty/OpenConsole.exe",
523 "node_modules/node-pty/prebuilds/win32-arm64/conpty/OpenConsole.exe",
524 "node_modules/node-pty/prebuilds/linux-x64/pty.node",
525 "node_modules/node-pty/third_party/conpty/1.0.0/win10-x64/OpenConsole.exe",
526 "node_modules/node-pty/third_party/conpty/1.0.0/win10-arm64/OpenConsole.exe",
527 ];
528
529 for (const nativeFile of nativeFiles) {
530 const nativePath = path.join(sourceDir, nativeFile);
531 yield* fs.makeDirectory(path.dirname(nativePath), { recursive: true });
532 yield* fs.writeFileString(nativePath, "native");
533 }
534
535 const asarPath = path.join(tempDir, "server.asar");
536 yield* packWindowsServerAsar({ sourceDir, asarPath, arch: "x64" });
537 const unpackedRoot = `${asarPath}.unpacked`;
538
539 assert.isTrue(
540 yield* fs.exists(
541 path.join(
542 unpackedRoot,
543 "node_modules/node-pty/prebuilds/win32-x64/conpty/OpenConsole.exe",
544 ),
545 ),
546 );
547 assert.isTrue(
548 yield* fs.exists(
549 path.join(unpackedRoot, "node_modules/node-pty/prebuilds/linux-x64/pty.node"),
550 ),
551 );
552 assert.isFalse(
553 yield* fs.exists(
554 path.join(unpackedRoot, "node_modules/node-pty/prebuilds/win32-arm64"),
555 ),
556 );
557 assert.isFalse(
558 yield* fs.exists(
559 path.join(unpackedRoot, "node_modules/node-pty/third_party/conpty/1.0.0/win10-arm64"),
560 ),
561 );
562 }),
563 ),
564 );
565
566 it.effect("stages a cached resource monitor without invoking Cargo", () =>
567 Effect.scoped(
568 Effect.gen(function* () {
569 const fs = yield* FileSystem.FileSystem;
570 const path = yield* Path.Path;
571 const repoRoot = yield* fs.makeTempDirectoryScoped({
572 prefix: "t3-resource-monitor-cache-test-",
573 });
574 const binaryPath = path.join(
575 repoRoot,
576 "native/resource-monitor/target/x86_64-unknown-linux-gnu/release/t3-resource-monitor",
577 );
578 const stageResourcesDir = path.join(repoRoot, "stage");
579 yield* fs.makeDirectory(path.dirname(binaryPath), { recursive: true });
580 yield* fs.writeFileString(binaryPath, "cached monitor");
581
582 yield* stageResourceMonitor({
583 repoRoot,
584 stageResourcesDir,
585 platform: "linux",
586 arch: "x64",
587 verbose: false,
588 }).pipe(
589 Effect.provide(
590 ConfigProvider.layer(
591 ConfigProvider.fromEnv({
592 env: { T3CODE_DESKTOP_REUSE_RESOURCE_MONITOR: "true" },
593 }),
594 ),
595 ),
596 );
597
598 assert.equal(
599 yield* fs.readFileString(
600 path.join(stageResourcesDir, "resource-monitor/t3-resource-monitor"),
601 ),
602 "cached monitor",
603 );
604 }),
605 ),
606 );
607
491 it.effect("validates every ASAR-unpacked native in the packaged Windows payload", () =>608 it.effect("validates every ASAR-unpacked native in the packaged Windows payload", () =>
492 Effect.scoped(609 Effect.scoped(
493 Effect.gen(function* () {610 Effect.gen(function* () {
@@ -504,6 +621,7 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { · 10 unmodified lines
504 yield* packWindowsServerAsar({621 yield* packWindowsServerAsar({
505 sourceDir: fixture.sourceDir,622 sourceDir: fixture.sourceDir,
506 asarPath: secondAsarPath,623 asarPath: secondAsarPath,
624 arch: "x64",
507 });625 });
508 const [firstAsar, secondAsar] = yield* Effect.all([626 const [firstAsar, secondAsar] = yield* Effect.all([
509 fs.readFile(fixture.generatedAsarPath),627 fs.readFile(fixture.generatedAsarPath),
scripts/build-desktop-artifact.ts+50−22
@@ -819,6 +819,21 @@ export const WINDOWS_SERVER_ASAR_IGNORE_GLOBS = [ · 818 unmodified lines
819 "**/node_modules/.bin",819 "**/node_modules/.bin",
820 "**/node_modules/.bin/**",820 "**/node_modules/.bin/**",
821] as const;821] as const;
822
823export function resolveWindowsServerAsarIgnoreGlobs(arch: typeof BuildArch.Type) {
824 const unusedArch = arch === "arm64" ? "x64" : "arm64";
825 const unusedPrebuild = `**/node_modules/node-pty/prebuilds/win32-${unusedArch}`;
826 const unusedConpty = `**/node_modules/node-pty/third_party/conpty/*/win10-${unusedArch}`;
827
828 return [
829 ...WINDOWS_SERVER_ASAR_IGNORE_GLOBS,
830 unusedPrebuild,
831 `${unusedPrebuild}/**`,
832 unusedConpty,
833 `${unusedConpty}/**`,
834 ];
835}
836
822export const WINDOWS_PACKAGED_PAYLOAD_FILE_LIMIT = 80;837export const WINDOWS_PACKAGED_PAYLOAD_FILE_LIMIT = 80;
823export const WINDOWS_SERVER_RESOURCE_SOURCE_DIR = "apps/desktop/prod-resources/windows-server";838export const WINDOWS_SERVER_RESOURCE_SOURCE_DIR = "apps/desktop/prod-resources/windows-server";
824export const WINDOWS_SERVER_EXTRA_RESOURCES = [839export const WINDOWS_SERVER_EXTRA_RESOURCES = [
@@ -1643,7 +1658,7 @@ const verifyPackagedBundleIsSelfContained = Effect.fn("verifyPackagedBundleIsSel · 818 unmodified lines
1643 },1658 },
1644);1659);
1645 1660
1646const stageResourceMonitor = Effect.fn("stageResourceMonitor")(function* (input: {1661export const stageResourceMonitor = Effect.fn("stageResourceMonitor")(function* (input: {
1647 readonly repoRoot: string;1662 readonly repoRoot: string;
1648 readonly stageResourcesDir: string;1663 readonly stageResourcesDir: string;
1649 readonly platform: typeof BuildPlatform.Type;1664 readonly platform: typeof BuildPlatform.Type;
@@ -1655,28 +1670,33 @@ const stageResourceMonitor = Effect.fn("stageResourceMonitor")(function* (input: · 5 unmodified lines
1655 const manifestPath = path.join(input.repoRoot, "native/resource-monitor/Cargo.toml");1670 const manifestPath = path.join(input.repoRoot, "native/resource-monitor/Cargo.toml");
1656 const executableName = resourceMonitorExecutableName(input.platform);1671 const executableName = resourceMonitorExecutableName(input.platform);
1657 const rustTargets = resolveResourceMonitorRustTargets(input.platform, input.arch);1672 const rustTargets = resolveResourceMonitorRustTargets(input.platform, input.arch);
1673 const reuseResourceMonitor = yield* Config.boolean("T3CODE_DESKTOP_REUSE_RESOURCE_MONITOR").pipe(
1674 Config.withDefault(false),
1675 );
1658 const builtBinaries: string[] = [];1676 const builtBinaries: string[] = [];
1659 1677
1660 for (const rustTarget of rustTargets) {1678 for (const rustTarget of rustTargets) {
1661 const spawnCommand = yield* resolveSpawnCommand("cargo", [1679 if (!reuseResourceMonitor) {
1662 "build",1680 const spawnCommand = yield* resolveSpawnCommand("cargo", [
1663 "--locked",1681 "build",
1664 "--release",1682 "--locked",
1665 "--manifest-path",1683 "--release",
1666 manifestPath,1684 "--manifest-path",
1667 "--target",1685 manifestPath,
1668 rustTarget,1686 "--target",
1669 ]);1687 rustTarget,
1670 yield* runCommand(1688 ]);
1671 ChildProcess.make(spawnCommand.command, spawnCommand.args, {1689 yield* runCommand(
1672 cwd: input.repoRoot,1690 ChildProcess.make(spawnCommand.command, spawnCommand.args, {
1673 shell: spawnCommand.shell,1691 cwd: input.repoRoot,
1674 }),1692 shell: spawnCommand.shell,
1675 {1693 }),
1676 label: `cargo build resource monitor (${rustTarget})`,1694 {
1677 verbose: input.verbose,1695 label: `cargo build resource monitor (${rustTarget})`,
1678 },1696 verbose: input.verbose,
1679 );1697 },
1698 );
1699 }
1680 1700
1681 const binaryPath = path.join(1701 const binaryPath = path.join(
1682 input.repoRoot,1702 input.repoRoot,
@@ -1693,6 +1713,9 @@ const stageResourceMonitor = Effect.fn("stageResourceMonitor")(function* (input: · 10 unmodified lines
1693 arch: input.arch,1713 arch: input.arch,
1694 });1714 });
1695 }1715 }
1716 if (reuseResourceMonitor) {
1717 yield* Effect.log(`[desktop-artifact] Reusing cached resource monitor (${rustTarget}).`);
1718 }
1696 builtBinaries.push(binaryPath);1719 builtBinaries.push(binaryPath);
1697 }1720 }
1698 1721
@@ -2253,6 +2276,7 @@ const stageWslNodePtyPrebuild = Effect.fn("stageWslNodePtyPrebuild")(function* ( · 554 unmodified lines
2253export const packWindowsServerAsar = Effect.fn("packWindowsServerAsar")(function* (input: {2276export const packWindowsServerAsar = Effect.fn("packWindowsServerAsar")(function* (input: {
2254 readonly sourceDir: string;2277 readonly sourceDir: string;
2255 readonly asarPath: string;2278 readonly asarPath: string;
2279 readonly arch: typeof BuildArch.Type;
2256}) {2280}) {
2257 const fs = yield* FileSystem.FileSystem;2281 const fs = yield* FileSystem.FileSystem;
2258 yield* Effect.tryPromise({2282 yield* Effect.tryPromise({
@@ -2260,7 +2284,7 @@ export const packWindowsServerAsar = Effect.fn("packWindowsServerAsar")(function · 1 unmodified line
2260 createPackageWithOptions(input.sourceDir, input.asarPath, {2284 createPackageWithOptions(input.sourceDir, input.asarPath, {
2261 dot: true,2285 dot: true,
2262 unpack: WINDOWS_SERVER_ASAR_UNPACK_GLOB,2286 unpack: WINDOWS_SERVER_ASAR_UNPACK_GLOB,
2263 globOptions: { ignore: [...WINDOWS_SERVER_ASAR_IGNORE_GLOBS] },2287 globOptions: { ignore: resolveWindowsServerAsarIgnoreGlobs(input.arch) },
2264 }),2288 }),
2265 catch: (cause) => new WindowsServerSidecarPackError({ asarPath: input.asarPath, cause }),2289 catch: (cause) => new WindowsServerSidecarPackError({ asarPath: input.asarPath, cause }),
2266 });2290 });
@@ -2354,7 +2378,11 @@ export const stageWindowsServerSidecar = Effect.fn("stageWindowsServerSidecar")( · 87 unmodified lines
2354 2378
2355 yield* Effect.log("[desktop-artifact] Packing server.asar...");2379 yield* Effect.log("[desktop-artifact] Packing server.asar...");
2356 yield* fs.makeDirectory(path.dirname(input.asarPath), { recursive: true });2380 yield* fs.makeDirectory(path.dirname(input.asarPath), { recursive: true });
2357 yield* packWindowsServerAsar({ sourceDir: serverStageDir, asarPath: input.asarPath });2381 yield* packWindowsServerAsar({
2382 sourceDir: serverStageDir,
2383 asarPath: input.asarPath,
2384 arch: input.arch,
2385 });
2358 const packedStat = yield* fs.stat(input.asarPath);2386 const packedStat = yield* fs.stat(input.asarPath);
2359 yield* Effect.log(2387 yield* Effect.log(
2360 `[desktop-artifact] Packed server.asar (${String(packedStat.size)} bytes) + unpacked natives.`,2388 `[desktop-artifact] Packed server.asar (${String(packedStat.size)} bytes) + unpacked natives.`,