History

compat with legacy dir

Rhys Sullivan committed 8 months ago

Commit a802f4e1 parent ed7a3486 files · +377 −25Browse files

Unified · Split

6 files
  1. .github/workflows/publish-executor-package.yml +72 −6
  2. apps/executor/package.json +2 −1
  3. apps/executor/src/distribution/build-release.ts +137 −0
  4. apps/executor/src/distribution/portable.ts +2 −0
  5. packages/server/src/config.ts +19 −0
  6. packages/server/src/index.ts +145 −18
.github/workflows/publish-executor-package.yml+72−6
@@ -5,7 +5,7 @@ on: · 4 unmodified lines
55 workflow_dispatch:
66
77 permissions:
8− contents: read
8+ contents: write
99 id-token: write
1010
1111 concurrency:
@@ -19,6 +19,8 @@ jobs: · 7 unmodified lines
1919 steps:
2020 - name: Checkout
2121 uses: actions/checkout@v4
22+ with:
23+ fetch-depth: 0
2224
2325 - name: Require default branch
2426 if: github.ref_name != github.event.repository.default_branch
@@ -53,15 +55,50 @@ jobs: · 28 unmodified lines
5355 fi
5456
5557 echo "version=${version}" >> "$GITHUB_OUTPUT"
58+ echo "tag=v${version}" >> "$GITHUB_OUTPUT"
5659 echo "Publishing executor@${version}"
5760
58− - name: Ensure version is unpublished
61+ - name: Inspect existing publish state
62+ id: state
5963 shell: bash
64+ env:
65+ GH_TOKEN: ${{ github.token }}
6066 run: |
6167 set -euo pipefail
68+ version="${{ steps.version.outputs.version }}"
69+ tag="${{ steps.version.outputs.tag }}"
70+ package_published=false
71+ release_exists=false
72+ tag_exists=false
73+
74+ if npm view "executor@${version}" version >/dev/null 2>&1; then
75+ package_published=true
76+ fi
77+
78+ if gh release view "${tag}" --repo "${GITHUB_REPOSITORY}" >/dev/null 2>&1; then
79+ release_exists=true
80+ fi
81+
82+ if git rev-parse -q --verify "refs/tags/${tag}" >/dev/null || git ls-remote --exit-code --tags origin "refs/tags/${tag}" >/dev/null 2>&1; then
83+ tag_exists=true
84+ fi
85+
86+ echo "package_published=${package_published}" >> "$GITHUB_OUTPUT"
87+ echo "release_exists=${release_exists}" >> "$GITHUB_OUTPUT"
88+ echo "tag_exists=${tag_exists}" >> "$GITHUB_OUTPUT"
89+
90+ if [[ "${package_published}" == true && "${release_exists}" == true ]]; then
91+ echo "executor@${version} is already published and ${tag} already exists" >&2
92+ exit 1
93+ fi
6294
63− if npm view "executor@${{ steps.version.outputs.version }}" version >/dev/null 2>&1; then
64− echo "executor@${{ steps.version.outputs.version }} is already published" >&2
95+ if [[ "${package_published}" == false && "${release_exists}" == true ]]; then
96+ echo "GitHub release ${tag} exists, but executor@${version} is not published to npm" >&2
97+ exit 1
98+ fi
99+
100+ if [[ "${package_published}" == false && "${tag_exists}" == true && "${release_exists}" == false ]]; then
101+ echo "Release tag ${tag} already exists without a GitHub release; resolve that before publishing executor@${version}" >&2
65102 exit 1
66103 fi
67104
@@ -70,9 +107,38 @@ jobs: · 2 unmodified lines
70107 bun run --cwd apps/executor typecheck
71108 bun run --cwd apps/executor test
72109
73− - name: Build distribution package
74− run: bun run --cwd apps/executor build:dist
110+ - name: Build release artifacts
111+ run: bun run --cwd apps/executor build:release
75112
76113 - name: Publish to npm
114+ if: steps.state.outputs.package_published != 'true'
77115 working-directory: apps/executor
78116 run: npm publish ./dist/npm --access public --provenance
117+
118+ - name: Create GitHub release
119+ if: steps.state.outputs.release_exists != 'true'
120+ shell: bash
121+ env:
122+ GH_TOKEN: ${{ github.token }}
123+ run: |
124+ set -euo pipefail
125+ shopt -s nullglob
126+ assets=(
127+ apps/executor/dist/release/*.tgz
128+ apps/executor/dist/release/checksums.txt
129+ apps/executor/dist/release/portable/*.tar.gz
130+ apps/executor/dist/release/portable/*.zip
131+ )
132+
133+ if [[ ${#assets[@]} -lt 3 ]]; then
134+ echo "Missing release assets under apps/executor/dist/release" >&2
135+ exit 1
136+ fi
137+
138+ gh release create "${{ steps.version.outputs.tag }}" \
139+ "${assets[@]}" \
140+ --repo "${GITHUB_REPOSITORY}" \
141+ --target "${GITHUB_SHA}" \
142+ --title "${{ steps.version.outputs.tag }}" \
143+ --generate-notes \
144+ --latest
apps/executor/package.json+2−1
@@ -33,7 +33,8 @@ · 32 unmodified lines
3333 "pack:dist": "bun src/distribution/build-package.ts && npm pack ./dist/npm",
3434 "publish:dist:dry-run": "bun src/distribution/build-package.ts && npm publish ./dist/npm --dry-run",
3535 "publish:dist": "bun src/distribution/build-package.ts && npm publish ./dist/npm",
36− "build:portable": "bun src/distribution/build-portable.ts"
36+ "build:portable": "bun src/distribution/build-portable.ts",
37+ "build:release": "bun src/distribution/build-release.ts"
3738 },
3839 "dependencies": {
3940 "@effect/cli": "catalog:",
apps/executor/src/distribution/build-release.ts+137−0
@@ -0,0 +1,137 @@
1+import { createHash } from "node:crypto";
2+import { spawn } from "node:child_process";
3+import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
4+import { join, relative, resolve } from "node:path";
5+
6+import { buildDistributionPackage } from "./artifact";
7+import { repoRoot } from "./metadata";
8+import { buildPortableDistribution, portableTargetIds } from "./portable";
9+
10+const defaultReleaseDir = resolve(repoRoot, "apps/executor/dist/release");
11+
12+type CommandInput = {
13+ command: string;
14+ args: ReadonlyArray<string>;
15+ cwd: string;
16+};
17+
18+type CommandOutput = {
19+ stdout: string;
20+ stderr: string;
21+};
22+
23+type PackResult = {
24+ filename?: string;
25+};
26+
27+const runCommand = async (input: CommandInput): Promise<CommandOutput> => {
28+ const child = spawn(input.command, [...input.args], {
29+ cwd: input.cwd,
30+ env: process.env,
31+ stdio: ["ignore", "pipe", "pipe"],
32+ });
33+
34+ let stdout = "";
35+ let stderr = "";
36+
37+ child.stdout?.setEncoding("utf8");
38+ child.stdout?.on("data", (chunk) => {
39+ stdout += chunk;
40+ });
41+
42+ child.stderr?.setEncoding("utf8");
43+ child.stderr?.on("data", (chunk) => {
44+ stderr += chunk;
45+ });
46+
47+ const exitCode = await new Promise<number>((resolveExitCode, reject) => {
48+ child.once("error", reject);
49+ child.once("close", (code) => {
50+ resolveExitCode(code ?? -1);
51+ });
52+ });
53+
54+ if (exitCode !== 0) {
55+ throw new Error(
56+ [
57+ `${input.command} ${input.args.join(" ")} exited with code ${exitCode}`,
58+ stdout.trim().length > 0 ? `stdout:\n${stdout.trim()}` : null,
59+ stderr.trim().length > 0 ? `stderr:\n${stderr.trim()}` : null,
60+ ]
61+ .filter((part) => part !== null)
62+ .join("\n\n"),
63+ );
64+ }
65+
66+ return {
67+ stdout: stdout.trim(),
68+ stderr: stderr.trim(),
69+ };
70+};
71+
72+const computeSha256 = (contents: Uint8Array): string =>
73+ createHash("sha256").update(contents).digest("hex");
74+
75+const createChecksumsFile = async (
76+ releaseDir: string,
77+ assetPaths: ReadonlyArray<string>,
78+): Promise<string> => {
79+ const entries = await Promise.all(
80+ [...assetPaths]
81+ .sort((left, right) => left.localeCompare(right))
82+ .map(async (assetPath) => {
83+ const contents = await readFile(assetPath);
84+ const relativePath = relative(releaseDir, assetPath).replaceAll("\\", "/");
85+ return `${computeSha256(contents)} ${relativePath}`;
86+ }),
87+ );
88+
89+ const checksumsPath = join(releaseDir, "checksums.txt");
90+ await writeFile(checksumsPath, `${entries.join("\n")}\n`);
91+ return checksumsPath;
92+};
93+
94+const packDistributionPackage = async (releaseDir: string): Promise<string> => {
95+ const output = await runCommand({
96+ command: "npm",
97+ args: ["pack", "./apps/executor/dist/npm", "--pack-destination", releaseDir, "--json"],
98+ cwd: repoRoot,
99+ });
100+ const [result] = JSON.parse(output.stdout) as ReadonlyArray<PackResult>;
101+ const filename = result?.filename;
102+
103+ if (!filename) {
104+ throw new Error(`npm pack did not report an output filename. stdout:\n${output.stdout}`);
105+ }
106+
107+ return join(releaseDir, filename);
108+};
109+
110+const main = async () => {
111+ const releaseDir = defaultReleaseDir;
112+ const portableDir = join(releaseDir, "portable");
113+
114+ await rm(releaseDir, { recursive: true, force: true });
115+ await mkdir(releaseDir, { recursive: true });
116+
117+ await buildDistributionPackage();
118+ const packageArchivePath = await packDistributionPackage(releaseDir);
119+ const portableArtifact = await buildPortableDistribution({
120+ outputDir: portableDir,
121+ buildWeb: false,
122+ targets: portableTargetIds,
123+ });
124+ const portableArchivePaths = portableArtifact.artifacts
125+ .map((artifact) => artifact.archivePath)
126+ .filter((archivePath): archivePath is string => archivePath !== null);
127+ const checksumsPath = await createChecksumsFile(releaseDir, [
128+ packageArchivePath,
129+ ...portableArchivePaths,
130+ ]);
131+
132+ for (const assetPath of [packageArchivePath, ...portableArchivePaths, checksumsPath]) {
133+ process.stdout.write(`${assetPath}\n`);
134+ }
135+};
136+
137+await main();
apps/executor/src/distribution/portable.ts+2−0
@@ -74,6 +74,8 @@ const portableTargets: Record<string, PortableTarget> = { · 73 unmodified lines
7474 },
7575 };
7676
77+export const portableTargetIds = Object.freeze(Object.keys(portableTargets));
78+
7779 export type BuildPortableDistributionOptions = {
7880 outputDir?: string;
7981 packageName?: string;
packages/server/src/config.ts+19−0
@@ -23,6 +23,17 @@ const defaultLinuxStateHome = trim(process.env.XDG_STATE_HOME) ?? join(home, ".l · 22 unmodified lines
2323 const defaultWindowsHome = join(trim(process.env.LOCALAPPDATA) ?? join(home, "AppData", "Local"), "Executor");
2424 const defaultMacHome = join(home, "Library", "Application Support", "Executor");
2525 const defaultLinuxHome = join(defaultLinuxDataHome, "executor");
26+const legacyLinuxHome = join(defaultLinuxDataHome, "executor-v3");
27+const legacyWindowsHome = join(
28+ trim(process.env.LOCALAPPDATA) ?? join(home, "AppData", "Local"),
29+ "Executor-v3",
30+);
31+const legacyMacHome = join(
32+ home,
33+ "Library",
34+ "Application Support",
35+ "Executor-v3",
36+);
2637
2738 export const DEFAULT_SERVER_PORT = 8788;
2839 export const DEFAULT_SERVER_HOST = "127.0.0.1";
@@ -43,6 +54,14 @@ export const DEFAULT_LOCAL_DATA_DIR = · 14 unmodified lines
4354 trim(process.env[EXECUTOR_LOCAL_DATA_DIR_ENV])
4455 ?? join(DEFAULT_EXECUTOR_DATA_DIR, "control-plane");
4556
57+export const DEFAULT_LEGACY_LOCAL_DATA_DIRS = explicitExecutorHome
58+ ? []
59+ : [
60+ isWindows ? join(legacyWindowsHome, "control-plane") : null,
61+ isMac ? join(legacyMacHome, "control-plane") : null,
62+ !isWindows && !isMac ? join(legacyLinuxHome, "control-plane") : null,
63+ ].filter((value): value is string => value !== null);
64+
4665 export const DEFAULT_SERVER_PID_FILE =
4766 trim(process.env[EXECUTOR_SERVER_PID_FILE_ENV])
4867 ?? join(DEFAULT_EXECUTOR_RUN_DIR, "server.pid");
packages/server/src/index.ts+145−18
@@ -1,13 +1,13 @@
1+import { existsSync } from "node:fs";
2+import { mkdir, readFile, rename, rm, stat, writeFile } from "node:fs/promises";
13 import {
24 createServer as createNodeServer,
35 type IncomingMessage,
46 type ServerResponse,
57 } from "node:http";
68 import type { AddressInfo } from "node:net";
7−import { mkdir, readFile, rm, stat, writeFile } from "node:fs/promises";
8−import { Readable } from "node:stream";
99 import { dirname, extname, resolve } from "node:path";
10−
10+import { Readable } from "node:stream";
1111 import { HttpApiBuilder, HttpServer } from "@effect/platform";
1212 import {
1313 createControlPlaneApiLayer,
@@ -17,11 +17,14 @@ import { · 3 unmodified lines
1717 type SqlControlPlaneRuntime,
1818 } from "@executor/control-plane";
1919 import { createExecutorMcpRequestHandler } from "@executor/executor-mcp";
20+import * as Cause from "effect/Cause";
2021 import * as Effect from "effect/Effect";
22+import * as Exit from "effect/Exit";
2123 import * as Layer from "effect/Layer";
2224 import type * as Scope from "effect/Scope";
2325
2426 import {
27+ DEFAULT_LEGACY_LOCAL_DATA_DIRS,
2528 DEFAULT_LOCAL_DATA_DIR,
2629 DEFAULT_SERVER_LOG_FILE,
2730 DEFAULT_SERVER_PID_FILE,
@@ -79,6 +82,9 @@ export type LocalExecutorRequestHandler = { · 51 unmodified lines
7982 readonly setBaseUrl: (baseUrl: string) => void;
8083 };
8184
85+type ControlPlaneWebHandler = ReturnType<typeof HttpApiBuilder.toWebHandler>;
86+type ExecutorMcpHandler = ReturnType<typeof createExecutorMcpRequestHandler>;
87+
8288 const disposeRuntime = (runtime: SqlControlPlaneRuntime) =>
8389 Effect.tryPromise({
8490 try: () => runtime.close(),
@@ -86,6 +92,121 @@ const disposeRuntime = (runtime: SqlControlPlaneRuntime) => · 1 unmodified line
8692 cause instanceof Error ? cause : new Error(String(cause ?? "runtime close failed")),
8793 }).pipe(Effect.orDie);
8894
95+const createRuntime = (
96+ localDataDir: string,
97+ getLocalServerBaseUrl: () => string | undefined,
98+ options: StartLocalExecutorServerOptions,
99+) =>
100+ createSqlControlPlaneRuntime({
101+ localDataDir,
102+ migrationsFolder: options.migrationsFolder,
103+ executionResolver: options.executionResolver,
104+ resolveSecretMaterial: options.resolveSecretMaterial,
105+ getLocalServerBaseUrl,
106+ }).pipe(
107+ Effect.mapError((cause) =>
108+ cause instanceof Error ? cause : new Error(String(cause)),
109+ ),
110+ );
111+
112+const moveLegacyLocalDataDir = (
113+ legacyLocalDataDir: string,
114+ requestedLocalDataDir: string,
115+ ) =>
116+ Effect.tryPromise({
117+ try: async () => {
118+ await mkdir(dirname(requestedLocalDataDir), { recursive: true });
119+ if (existsSync(requestedLocalDataDir)) {
120+ const backupPath = `${requestedLocalDataDir}.backup-${Date.now()}`;
121+ await rename(requestedLocalDataDir, backupPath);
122+ console.warn(
123+ `[executor] Backed up unreadable local data dir to: ${backupPath}`,
124+ );
125+ }
126+ await rename(legacyLocalDataDir, requestedLocalDataDir);
127+ console.warn(
128+ `[executor] Moved legacy local data dir to: ${requestedLocalDataDir}`,
129+ );
130+ },
131+ catch: (cause) =>
132+ cause instanceof Error ? cause : new Error(String(cause)),
133+ });
134+
135+const createRuntimeWithLegacyMigration = (
136+ requestedLocalDataDir: string,
137+ legacyLocalDataDirs: ReadonlyArray<string>,
138+ getLocalServerBaseUrl: () => string | undefined,
139+ options: StartLocalExecutorServerOptions,
140+ ) =>
141+ Effect.gen(function* () {
142+ if (
143+ requestedLocalDataDir !== ":memory:"
144+ && !existsSync(requestedLocalDataDir)
145+ && legacyLocalDataDirs.length > 0
146+ ) {
147+ for (const legacyLocalDataDir of legacyLocalDataDirs) {
148+ const legacyExit = yield* Effect.exit(
149+ createRuntime(legacyLocalDataDir, getLocalServerBaseUrl, options),
150+ );
151+ if (Exit.isFailure(legacyExit)) {
152+ continue;
153+ }
154+ yield* disposeRuntime(legacyExit.value);
155+ const migrationExit = yield* Effect.exit(
156+ moveLegacyLocalDataDir(legacyLocalDataDir, requestedLocalDataDir),
157+ );
158+ if (Exit.isFailure(migrationExit)) {
159+ continue;
160+ }
161+ const migratedExit = yield* Effect.exit(
162+ createRuntime(requestedLocalDataDir, getLocalServerBaseUrl, options),
163+ );
164+ if (Exit.isSuccess(migratedExit)) {
165+ return migratedExit.value;
166+ }
167+ }
168+ }
169+ const primaryExit = yield* Effect.exit(
170+ createRuntime(requestedLocalDataDir, getLocalServerBaseUrl, options),
171+ );
172+ if (Exit.isSuccess(primaryExit)) {
173+ return primaryExit.value;
174+ }
175+ const primaryError = Cause.squash(primaryExit.cause);
176+ if (legacyLocalDataDirs.length > 0) {
177+ yield* Effect.sync(() => {
178+ console.warn(
179+ `[executor] Failed to open default local data dir: ${requestedLocalDataDir}`,
180+ primaryError instanceof Error ? primaryError.message : String(primaryError),
181+ );
182+ });
183+ }
184+ for (const legacyLocalDataDir of legacyLocalDataDirs) {
185+ const legacyExit = yield* Effect.exit(
186+ createRuntime(legacyLocalDataDir, getLocalServerBaseUrl, options),
187+ );
188+ if (Exit.isFailure(legacyExit)) {
189+ continue;
190+ }
191+ yield* disposeRuntime(legacyExit.value);
192+ const migrationExit = yield* Effect.exit(
193+ moveLegacyLocalDataDir(legacyLocalDataDir, requestedLocalDataDir),
194+ );
195+ if (Exit.isFailure(migrationExit)) {
196+ continue;
197+ }
198+ const migratedExit = yield* Effect.exit(
199+ createRuntime(requestedLocalDataDir, getLocalServerBaseUrl, options),
200+ );
201+ if (Exit.isSuccess(migratedExit)) {
202+ return migratedExit.value;
203+ }
204+ }
205+ return yield* Effect.fail(
206+ primaryError instanceof Error ? primaryError : new Error(String(primaryError)),
207+ );
208+ });
209+
89210 const createControlPlaneWebHandler = (runtime: SqlControlPlaneRuntime) =>
90211 Effect.acquireRelease(
91212 Effect.sync(() =>
@@ -96,7 +217,7 @@ const createControlPlaneWebHandler = (runtime: SqlControlPlaneRuntime) => · 4 unmodified lines
96217 ),
97218 ),
98219 ),
99− (handler) => Effect.tryPromise({ try: () => handler.dispose(), catch: (cause) => cause instanceof Error ? cause : new Error(String(cause ?? "web handler dispose failed")) }).pipe(Effect.orDie),
220+ (handler: ControlPlaneWebHandler) => Effect.tryPromise({ try: () => handler.dispose(), catch: (cause) => cause instanceof Error ? cause : new Error(String(cause ?? "web handler dispose failed")) }).pipe(Effect.orDie),
100221 );
101222
102223 const safeFilePath = (assetsDir: string, pathname: string): string | null => {
@@ -255,11 +376,15 @@ export const createLocalExecutorRequestHandler = ( · 152 unmodified lines
255376 options: StartLocalExecutorServerOptions = {},
256377 ): Effect.Effect<LocalExecutorRequestHandler, Error, Scope.Scope> =>
257378 Effect.gen(function* () {
258− const localDataDir = options.localDataDir ?? DEFAULT_LOCAL_DATA_DIR;
379+ const requestedLocalDataDir = options.localDataDir ?? DEFAULT_LOCAL_DATA_DIR;
380+ const legacyLocalDataDirs =
381+ options.localDataDir === undefined
382+ ? DEFAULT_LEGACY_LOCAL_DATA_DIRS.filter((candidate) => existsSync(candidate))
383+ : [];
259384
260− if (localDataDir !== ":memory:") {
385+ if (requestedLocalDataDir !== ":memory:") {
261386 yield* Effect.tryPromise({
262− try: () => mkdir(dirname(localDataDir), { recursive: true }),
387+ try: () => mkdir(dirname(requestedLocalDataDir), { recursive: true }),
263388 catch: (cause) => (cause instanceof Error ? cause : new Error(String(cause))),
264389 });
265390 }
@@ -267,16 +392,11 @@ export const createLocalExecutorRequestHandler = ( · 1 unmodified line
267392 let baseUrlRef: string | undefined;
268393
269394 const runtime = yield* Effect.acquireRelease(
270− createSqlControlPlaneRuntime({
271− localDataDir,
272− migrationsFolder: options.migrationsFolder,
273− executionResolver: options.executionResolver,
274− resolveSecretMaterial: options.resolveSecretMaterial,
275− getLocalServerBaseUrl: () => baseUrlRef,
276− }).pipe(
277− Effect.mapError((cause) =>
278− cause instanceof Error ? cause : new Error(String(cause)),
279− ),
395+ createRuntimeWithLegacyMigration(
396+ requestedLocalDataDir,
397+ legacyLocalDataDirs,
398+ () => baseUrlRef,
399+ options,
280400 ),
281401 disposeRuntime,
282402 );
@@ -284,7 +404,14 @@ export const createLocalExecutorRequestHandler = ( · 1 unmodified line
284404 const apiHandler = yield* createControlPlaneWebHandler(runtime);
285405 const mcpHandler = yield* Effect.acquireRelease(
286406 Effect.sync(() => createExecutorMcpRequestHandler(runtime)),
287− (handler) => Effect.tryPromise({ try: () => handler.close(), catch: (cause) => cause instanceof Error ? cause : new Error(String(cause ?? "mcp handler close failed")) }).pipe(Effect.orDie),
407+ (handler: ExecutorMcpHandler) =>
408+ Effect.tryPromise({
409+ try: () => handler.close(),
410+ catch: (cause) =>
411+ cause instanceof Error
412+ ? cause
413+ : new Error(String(cause ?? "mcp handler close failed")),
414+ }).pipe(Effect.orDie),
288415 );
289416
290417 return {