History

perf(ci): speed up release builds and Windows packaging (#7975)

Theo Browne committed 2 months ago

Commit 25dcee01 parent f70eeee6 files · +251 −120Browse files

Showing the changes to scripts (2 of 6 files). Show all

Unified · Split

2 files
  1. scripts/build-desktop-artifact.test.ts +119 −1
  2. scripts/build-desktop-artifact.ts +50 −22
scripts/build-desktop-artifact.test.ts+119−1
@@ -40,6 +40,7 @@ import { · 39 unmodified lines
4040 resolveDesktopUpdateChannel,
4141 resolveDesktopWebAssetBrand,
4242 resolveResourceMonitorRustTargets,
43+ resolveWindowsServerAsarIgnoreGlobs,
4344 resourceMonitorExecutableName,
4445 resolveGitHubPublishConfig,
4546 resolveMockUpdateServerPort,
@@ -47,6 +48,7 @@ import { · 1 unmodified line
4748 resolvePackageManagerUserAgent,
4849 stageLinuxIconSize,
4950 stageDesktopDmgBackground,
51+ stageResourceMonitor,
5052 STAGE_INSTALL_ARGS,
5153 ancestorNodeModulesPaths,
5254 copyDirectoryPreservingSymlinks,
@@ -118,7 +120,7 @@ const makeWindowsPayloadFixture = Effect.fn("test.makeWindowsPayloadFixture")(fu · 65 unmodified lines
118120 yield* fs.writeFileString(nativePath, "native-binary");
119121
120122 const generatedAsarPath = path.join(tempDir, WINDOWS_SERVER_ASAR_RESOURCE);
121− yield* packWindowsServerAsar({ sourceDir, asarPath: generatedAsarPath });
123+ yield* packWindowsServerAsar({ sourceDir, asarPath: generatedAsarPath, arch: "x64" });
122124
123125 const stageDistDir = path.join(tempDir, "dist");
124126 const packagedAppDir = path.join(stageDistDir, "win-unpacked");
@@ -488,6 +490,121 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { · 363 unmodified lines
488490 }).pipe(Effect.provide(ConfigProvider.layer(ConfigProvider.fromEnv({ env: {} })))),
489491 );
490492
493+ it("excludes node-pty binaries for the other Windows architecture", () => {
494+ assert.deepStrictEqual(resolveWindowsServerAsarIgnoreGlobs("x64"), [
495+ ...WINDOWS_SERVER_ASAR_IGNORE_GLOBS,
496+ "**/node_modules/node-pty/prebuilds/win32-arm64",
497+ "**/node_modules/node-pty/prebuilds/win32-arm64/**",
498+ "**/node_modules/node-pty/third_party/conpty/*/win10-arm64",
499+ "**/node_modules/node-pty/third_party/conpty/*/win10-arm64/**",
500+ ]);
501+ assert.deepStrictEqual(resolveWindowsServerAsarIgnoreGlobs("arm64"), [
502+ ...WINDOWS_SERVER_ASAR_IGNORE_GLOBS,
503+ "**/node_modules/node-pty/prebuilds/win32-x64",
504+ "**/node_modules/node-pty/prebuilds/win32-x64/**",
505+ "**/node_modules/node-pty/third_party/conpty/*/win10-x64",
506+ "**/node_modules/node-pty/third_party/conpty/*/win10-x64/**",
507+ ]);
508+ });
509+
510+ it.effect(
511+ "keeps target and WSL native files while excluding the other Windows architecture",
512+ () =>
513+ Effect.scoped(
514+ Effect.gen(function* () {
515+ const fs = yield* FileSystem.FileSystem;
516+ const path = yield* Path.Path;
517+ const tempDir = yield* fs.makeTempDirectoryScoped({
518+ prefix: "t3-windows-architecture-test-",
519+ });
520+ const sourceDir = path.join(tempDir, "server");
521+ const nativeFiles = [
522+ "node_modules/node-pty/prebuilds/win32-x64/conpty/OpenConsole.exe",
523+ "node_modules/node-pty/prebuilds/win32-arm64/conpty/OpenConsole.exe",
524+ "node_modules/node-pty/prebuilds/linux-x64/pty.node",
525+ "node_modules/node-pty/third_party/conpty/1.0.0/win10-x64/OpenConsole.exe",
526+ "node_modules/node-pty/third_party/conpty/1.0.0/win10-arm64/OpenConsole.exe",
527+ ];
528+
529+ for (const nativeFile of nativeFiles) {
530+ const nativePath = path.join(sourceDir, nativeFile);
531+ yield* fs.makeDirectory(path.dirname(nativePath), { recursive: true });
532+ yield* fs.writeFileString(nativePath, "native");
533+ }
534+
535+ const asarPath = path.join(tempDir, "server.asar");
536+ yield* packWindowsServerAsar({ sourceDir, asarPath, arch: "x64" });
537+ const unpackedRoot = `${asarPath}.unpacked`;
538+
539+ assert.isTrue(
540+ yield* fs.exists(
541+ path.join(
542+ unpackedRoot,
543+ "node_modules/node-pty/prebuilds/win32-x64/conpty/OpenConsole.exe",
544+ ),
545+ ),
546+ );
547+ assert.isTrue(
548+ yield* fs.exists(
549+ path.join(unpackedRoot, "node_modules/node-pty/prebuilds/linux-x64/pty.node"),
550+ ),
551+ );
552+ assert.isFalse(
553+ yield* fs.exists(
554+ path.join(unpackedRoot, "node_modules/node-pty/prebuilds/win32-arm64"),
555+ ),
556+ );
557+ assert.isFalse(
558+ yield* fs.exists(
559+ path.join(unpackedRoot, "node_modules/node-pty/third_party/conpty/1.0.0/win10-arm64"),
560+ ),
561+ );
562+ }),
563+ ),
564+ );
565+
566+ it.effect("stages a cached resource monitor without invoking Cargo", () =>
567+ Effect.scoped(
568+ Effect.gen(function* () {
569+ const fs = yield* FileSystem.FileSystem;
570+ const path = yield* Path.Path;
571+ const repoRoot = yield* fs.makeTempDirectoryScoped({
572+ prefix: "t3-resource-monitor-cache-test-",
573+ });
574+ const binaryPath = path.join(
575+ repoRoot,
576+ "native/resource-monitor/target/x86_64-unknown-linux-gnu/release/t3-resource-monitor",
577+ );
578+ const stageResourcesDir = path.join(repoRoot, "stage");
579+ yield* fs.makeDirectory(path.dirname(binaryPath), { recursive: true });
580+ yield* fs.writeFileString(binaryPath, "cached monitor");
581+
582+ yield* stageResourceMonitor({
583+ repoRoot,
584+ stageResourcesDir,
585+ platform: "linux",
586+ arch: "x64",
587+ verbose: false,
588+ }).pipe(
589+ Effect.provide(
590+ ConfigProvider.layer(
591+ ConfigProvider.fromEnv({
592+ env: { T3CODE_DESKTOP_REUSE_RESOURCE_MONITOR: "true" },
593+ }),
594+ ),
595+ ),
596+ );
597+
598+ assert.equal(
599+ yield* fs.readFileString(
600+ path.join(stageResourcesDir, "resource-monitor/t3-resource-monitor"),
601+ ),
602+ "cached monitor",
603+ );
604+ }),
605+ ),
606+ );
607+
491608 it.effect("validates every ASAR-unpacked native in the packaged Windows payload", () =>
492609 Effect.scoped(
493610 Effect.gen(function* () {
@@ -504,6 +621,7 @@ it.layer(NodeServices.layer)("build-desktop-artifact", (it) => { · 10 unmodified lines
504621 yield* packWindowsServerAsar({
505622 sourceDir: fixture.sourceDir,
506623 asarPath: secondAsarPath,
624+ arch: "x64",
507625 });
508626 const [firstAsar, secondAsar] = yield* Effect.all([
509627 fs.readFile(fixture.generatedAsarPath),
scripts/build-desktop-artifact.ts+50−22
@@ -819,6 +819,21 @@ export const WINDOWS_SERVER_ASAR_IGNORE_GLOBS = [ · 818 unmodified lines
819819 "**/node_modules/.bin",
820820 "**/node_modules/.bin/**",
821821 ] as const;
822+
823+export function resolveWindowsServerAsarIgnoreGlobs(arch: typeof BuildArch.Type) {
824+ const unusedArch = arch === "arm64" ? "x64" : "arm64";
825+ const unusedPrebuild = `**/node_modules/node-pty/prebuilds/win32-${unusedArch}`;
826+ const unusedConpty = `**/node_modules/node-pty/third_party/conpty/*/win10-${unusedArch}`;
827+
828+ return [
829+ ...WINDOWS_SERVER_ASAR_IGNORE_GLOBS,
830+ unusedPrebuild,
831+ `${unusedPrebuild}/**`,
832+ unusedConpty,
833+ `${unusedConpty}/**`,
834+ ];
835+}
836+
822837 export const WINDOWS_PACKAGED_PAYLOAD_FILE_LIMIT = 80;
823838 export const WINDOWS_SERVER_RESOURCE_SOURCE_DIR = "apps/desktop/prod-resources/windows-server";
824839 export const WINDOWS_SERVER_EXTRA_RESOURCES = [
@@ -1643,7 +1658,7 @@ const verifyPackagedBundleIsSelfContained = Effect.fn("verifyPackagedBundleIsSel · 818 unmodified lines
16431658 },
16441659 );
16451660
1646−const stageResourceMonitor = Effect.fn("stageResourceMonitor")(function* (input: {
1661+export const stageResourceMonitor = Effect.fn("stageResourceMonitor")(function* (input: {
16471662 readonly repoRoot: string;
16481663 readonly stageResourcesDir: string;
16491664 readonly platform: typeof BuildPlatform.Type;
@@ -1655,28 +1670,33 @@ const stageResourceMonitor = Effect.fn("stageResourceMonitor")(function* (input: · 5 unmodified lines
16551670 const manifestPath = path.join(input.repoRoot, "native/resource-monitor/Cargo.toml");
16561671 const executableName = resourceMonitorExecutableName(input.platform);
16571672 const rustTargets = resolveResourceMonitorRustTargets(input.platform, input.arch);
1673+ const reuseResourceMonitor = yield* Config.boolean("T3CODE_DESKTOP_REUSE_RESOURCE_MONITOR").pipe(
1674+ Config.withDefault(false),
1675+ );
16581676 const builtBinaries: string[] = [];
16591677
16601678 for (const rustTarget of rustTargets) {
1661− const spawnCommand = yield* resolveSpawnCommand("cargo", [
1662− "build",
1663− "--locked",
1664− "--release",
1665− "--manifest-path",
1666− manifestPath,
1667− "--target",
1668− rustTarget,
1669− ]);
1670− yield* runCommand(
1671− ChildProcess.make(spawnCommand.command, spawnCommand.args, {
1672− cwd: input.repoRoot,
1673− shell: spawnCommand.shell,
1674− }),
1675− {
1676− label: `cargo build resource monitor (${rustTarget})`,
1677− verbose: input.verbose,
1678− },
1679− );
1679+ if (!reuseResourceMonitor) {
1680+ const spawnCommand = yield* resolveSpawnCommand("cargo", [
1681+ "build",
1682+ "--locked",
1683+ "--release",
1684+ "--manifest-path",
1685+ manifestPath,
1686+ "--target",
1687+ rustTarget,
1688+ ]);
1689+ yield* runCommand(
1690+ ChildProcess.make(spawnCommand.command, spawnCommand.args, {
1691+ cwd: input.repoRoot,
1692+ shell: spawnCommand.shell,
1693+ }),
1694+ {
1695+ label: `cargo build resource monitor (${rustTarget})`,
1696+ verbose: input.verbose,
1697+ },
1698+ );
1699+ }
16801700
16811701 const binaryPath = path.join(
16821702 input.repoRoot,
@@ -1693,6 +1713,9 @@ const stageResourceMonitor = Effect.fn("stageResourceMonitor")(function* (input: · 10 unmodified lines
16931713 arch: input.arch,
16941714 });
16951715 }
1716+ if (reuseResourceMonitor) {
1717+ yield* Effect.log(`[desktop-artifact] Reusing cached resource monitor (${rustTarget}).`);
1718+ }
16961719 builtBinaries.push(binaryPath);
16971720 }
16981721
@@ -2253,6 +2276,7 @@ const stageWslNodePtyPrebuild = Effect.fn("stageWslNodePtyPrebuild")(function* ( · 554 unmodified lines
22532276 export const packWindowsServerAsar = Effect.fn("packWindowsServerAsar")(function* (input: {
22542277 readonly sourceDir: string;
22552278 readonly asarPath: string;
2279+ readonly arch: typeof BuildArch.Type;
22562280 }) {
22572281 const fs = yield* FileSystem.FileSystem;
22582282 yield* Effect.tryPromise({
@@ -2260,7 +2284,7 @@ export const packWindowsServerAsar = Effect.fn("packWindowsServerAsar")(function · 1 unmodified line
22602284 createPackageWithOptions(input.sourceDir, input.asarPath, {
22612285 dot: true,
22622286 unpack: WINDOWS_SERVER_ASAR_UNPACK_GLOB,
2263− globOptions: { ignore: [...WINDOWS_SERVER_ASAR_IGNORE_GLOBS] },
2287+ globOptions: { ignore: resolveWindowsServerAsarIgnoreGlobs(input.arch) },
22642288 }),
22652289 catch: (cause) => new WindowsServerSidecarPackError({ asarPath: input.asarPath, cause }),
22662290 });
@@ -2354,7 +2378,11 @@ export const stageWindowsServerSidecar = Effect.fn("stageWindowsServerSidecar")( · 87 unmodified lines
23542378
23552379 yield* Effect.log("[desktop-artifact] Packing server.asar...");
23562380 yield* fs.makeDirectory(path.dirname(input.asarPath), { recursive: true });
2357− yield* packWindowsServerAsar({ sourceDir: serverStageDir, asarPath: input.asarPath });
2381+ yield* packWindowsServerAsar({
2382+ sourceDir: serverStageDir,
2383+ asarPath: input.asarPath,
2384+ arch: input.arch,
2385+ });
23582386 const packedStat = yield* fs.stat(input.asarPath);
23592387 yield* Effect.log(
23602388 `[desktop-artifact] Packed server.asar (${String(packedStat.size)} bytes) + unpacked natives.`,