History

Fix local SQLite import retry

Rhys Sullivan committed 5 months agov1.4.31

Commit bc70f841 parent cb6c3f026 files · +126 −61Browse files

Unified · Split

26 files
  1. apps/cli/package.json +1 −1
  2. apps/cloud/package.json +1 −1
  3. apps/desktop/package.json +1 −1
  4. apps/local/src/server/executor.ts +26 −14
  5. apps/local/src/server/sqlite-import.test.ts +54 −1
  6. bun.lock +23 −23
  7. examples/all-plugins/package.json +1 −1
  8. examples/docs-sdk-quickstart/package.json +1 −1
  9. packages/core/api/package.json +1 −1
  10. packages/core/cli/package.json +1 −1
  11. packages/core/config/package.json +1 −1
  12. packages/core/execution/package.json +1 −1
  13. packages/core/sdk/package.json +1 −1
  14. packages/core/vite-plugin/package.json +1 −1
  15. packages/kernel/core/package.json +1 −1
  16. packages/kernel/runtime-quickjs/package.json +1 −1
  17. packages/plugins/desktop-settings/package.json +1 −1
  18. packages/plugins/example/package.json +1 −1
  19. packages/plugins/file-secrets/package.json +1 −1
  20. packages/plugins/google-discovery/package.json +1 −1
  21. packages/plugins/graphql/package.json +1 −1
  22. packages/plugins/keychain/package.json +1 −1
  23. packages/plugins/mcp/package.json +1 −1
  24. packages/plugins/onepassword/package.json +1 −1
  25. packages/plugins/openapi/package.json +1 −1
  26. packages/react/package.json +1 −1
apps/cli/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "executor",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "private": true,
55 "bin": {
66 "executor": "./bin/executor.ts"
apps/cloud/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/cloud",
3− "version": "1.4.16",
3+ "version": "1.4.17",
44 "private": true,
55 "type": "module",
66 "scripts": {
apps/desktop/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/desktop",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "private": true,
55 "homepage": "https://github.com/RhysSullivan/executor",
66 "license": "MIT",
apps/local/src/server/executor.ts+26−14
@@ -129,6 +129,7 @@ export type LocalExecutor = LocalExecutorBundle["executor"]; · 128 unmodified lines
129129
130130 class LocalExecutorCreateError extends Data.TaggedError("LocalExecutorCreateError")<{
131131 readonly operation: "createSqlite" | "importSqlite";
132+ readonly message: string;
132133 readonly cause: unknown;
133134 }> {}
134135
@@ -137,6 +138,19 @@ class LocalExecutorDisposeError extends Data.TaggedError("LocalExecutorDisposeEr · 2 unmodified lines
137138 readonly cause: unknown;
138139 }> {}
139140
141+const localExecutorCreateError = (
142+ operation: LocalExecutorCreateError["operation"],
143+ cause: unknown,
144+) =>
145+ new LocalExecutorCreateError({
146+ operation,
147+ cause,
148+ message:
149+ operation === "importSqlite"
150+ ? "Failed to prepare local SQLite data. Close other Executor processes and retry, or run with --log-level debug for details."
151+ : "Failed to open local SQLite data. Close other Executor processes and retry, or run with --log-level debug for details.",
152+ });
153+
140154 const ignorePromiseFailure = (
141155 operation: LocalExecutorDisposeError["operation"],
142156 try_: () => Promise<unknown>,
@@ -460,19 +474,17 @@ const importMissingMarkedTables = async (input: { · 317 unmodified lines
460474 await target.close();
461475 removeSqliteSidecars(input.storage.sqlitePath);
462476
463− if (result.imported) {
464− const importedTables = [
465− ...new Set([...input.marker.importedTables, ...result.importedTables]),
466− ];
467− writeSqliteImportMarker(input.storage.importMarkerPath, {
468− importedRows: input.marker.importedRows + result.importedRows,
469− importedTables,
470− backupPath: input.marker.backupPath,
471− recovered: input.marker.recovered,
472− });
473− }
477+ const importedTables = [...new Set([...input.marker.importedTables, ...missingTables])];
478+ writeSqliteImportMarker(input.storage.importMarkerPath, {
479+ importedRows: input.marker.importedRows + result.importedRows,
480+ importedTables,
481+ backupPath: input.marker.backupPath,
482+ recovered: input.marker.recovered,
483+ });
474484
475− return result;
485+ return result.importedRows > 0 || result.importedTables.length > 0
486+ ? result
487+ : { imported: false, importedRows: 0, importedTables: [] };
476488 } catch (cause) {
477489 await target.close();
478490 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: preserve late plugin-table import failure after closing SQLite
@@ -633,7 +645,7 @@ const createLocalExecutorLayer = () => { · 154 unmodified lines
633645 tables,
634646 scopeId,
635647 }),
636− catch: (cause) => new LocalExecutorCreateError({ operation: "importSqlite", cause }),
648+ catch: (cause) => localExecutorCreateError("importSqlite", cause),
637649 });
638650
639651 const sqlite = yield* Effect.acquireRelease(
@@ -644,7 +656,7 @@ const createLocalExecutorLayer = () => { · 4 unmodified lines
644656 namespace: localNamespace,
645657 path: storage.sqlitePath,
646658 }),
647− catch: (cause) => new LocalExecutorCreateError({ operation: "createSqlite", cause }),
659+ catch: (cause) => localExecutorCreateError("createSqlite", cause),
648660 }),
649661 (db) => Effect.promise(() => db.close()).pipe(Effect.ignore),
650662 );
apps/local/src/server/sqlite-import.test.ts+54−1
@@ -1,6 +1,7 @@
11 import { afterEach, beforeEach, describe, expect, it } from "@effect/vitest";
22 import { Database } from "bun:sqlite";
3−import { existsSync, mkdtempSync, rmSync } from "node:fs";
3+import { Schema } from "effect";
4+import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs";
45 import { tmpdir } from "node:os";
56 import { join } from "node:path";
67
@@ -166,6 +167,13 @@ const latePlugin = definePlugin(() => ({ · 159 unmodified lines
166167 storage: () => ({}),
167168 }))();
168169
170+const ImportMarkerForTest = Schema.Struct({
171+ importedTables: Schema.Array(Schema.String),
172+});
173+const decodeImportMarkerForTest = Schema.decodeUnknownSync(
174+ Schema.fromJsonString(ImportMarkerForTest),
175+);
176+
169177 const legacyShapeSchema = {
170178 legacy_shape: scopedExecutorTable("legacy_shape", {
171179 payload: jsonColumn("payload"),
@@ -497,4 +505,49 @@ describe("importSqliteDataToFuma", () => { · 325 unmodified lines
497505 }),
498506 ).resolves.toEqual([{ id: "late_1", value: "from-backup" }]);
499507 });
508+
509+ it("marks newly-loaded empty plugin tables so startup does not retry backup imports", async () => {
510+ const sqlitePath = join(workDir, "data.db");
511+ const markerPath = join(workDir, "fumadb-sqlite-imported");
512+ seedMigratedSqlite(sqlitePath);
513+
514+ const firstResult = await importLegacySqliteIfNeeded({
515+ storage: {
516+ dataDir: workDir,
517+ sqlitePath,
518+ importMarkerPath: markerPath,
519+ },
520+ tables: collectTables([]),
521+ scopeId: "scope_a",
522+ });
523+ expect(firstResult.importedTables).not.toContain("late_item");
524+
525+ const allTables = collectTables([latePlugin]);
526+ const secondResult = await importLegacySqliteIfNeeded({
527+ storage: {
528+ dataDir: workDir,
529+ sqlitePath,
530+ importMarkerPath: markerPath,
531+ },
532+ tables: allTables,
533+ scopeId: "scope_a",
534+ });
535+
536+ expect(secondResult.imported).toBe(false);
537+ expect(secondResult.importedRows).toBe(0);
538+ expect(decodeImportMarkerForTest(readFileSync(markerPath, "utf8")).importedTables).toContain(
539+ "late_item",
540+ );
541+
542+ const thirdResult = await importLegacySqliteIfNeeded({
543+ storage: {
544+ dataDir: workDir,
545+ sqlitePath,
546+ importMarkerPath: markerPath,
547+ },
548+ tables: allTables,
549+ scopeId: "scope_a",
550+ });
551+ expect(thirdResult.imported).toBe(false);
552+ });
500553 });
bun.lock+23−23
@@ -29,7 +29,7 @@ · 28 unmodified lines
2929 },
3030 "apps/cli": {
3131 "name": "executor",
32− "version": "1.4.30",
32+ "version": "1.4.31",
3333 "bin": {
3434 "executor": "./bin/executor.ts",
3535 },
@@ -50,7 +50,7 @@ · 14 unmodified lines
5050 },
5151 "apps/cloud": {
5252 "name": "@executor-js/cloud",
53− "version": "1.4.16",
53+ "version": "1.4.17",
5454 "dependencies": {
5555 "@cloudflare/vite-plugin": "^1.31.1",
5656 "@effect/atom-react": "catalog:",
@@ -117,7 +117,7 @@ · 60 unmodified lines
117117 },
118118 "apps/desktop": {
119119 "name": "@executor-js/desktop",
120− "version": "1.4.30",
120+ "version": "1.4.31",
121121 "dependencies": {
122122 "electron-log": "^5",
123123 "electron-store": "^10",
@@ -231,7 +231,7 @@ · 107 unmodified lines
231231 },
232232 "examples/all-plugins": {
233233 "name": "@executor-js/example-all-plugins",
234− "version": "0.0.16",
234+ "version": "0.0.17",
235235 "dependencies": {
236236 "@executor-js/plugin-file-secrets": "workspace:*",
237237 "@executor-js/plugin-google-discovery": "workspace:*",
@@ -251,7 +251,7 @@ · 13 unmodified lines
251251 },
252252 "examples/docs-sdk-quickstart": {
253253 "name": "@executor-js/example-docs-sdk-quickstart",
254− "version": "0.0.1",
254+ "version": "0.0.2",
255255 "dependencies": {
256256 "@executor-js/plugin-openapi": "workspace:*",
257257 "@executor-js/sdk": "workspace:*",
@@ -301,7 +301,7 @@ · 43 unmodified lines
301301 },
302302 "packages/core/api": {
303303 "name": "@executor-js/api",
304− "version": "1.4.18",
304+ "version": "1.4.19",
305305 "dependencies": {
306306 "@executor-js/execution": "workspace:*",
307307 "@executor-js/sdk": "workspace:*",
@@ -317,7 +317,7 @@ · 9 unmodified lines
317317 },
318318 "packages/core/cli": {
319319 "name": "@executor-js/cli",
320− "version": "0.2.2",
320+ "version": "0.2.3",
321321 "bin": {
322322 "executor-sdk": "./dist/index.js",
323323 },
@@ -338,7 +338,7 @@ · 14 unmodified lines
338338 },
339339 "packages/core/config": {
340340 "name": "@executor-js/config",
341− "version": "1.4.30",
341+ "version": "1.4.31",
342342 "dependencies": {
343343 "@executor-js/sdk": "workspace:*",
344344 "effect": "catalog:",
@@ -356,7 +356,7 @@ · 11 unmodified lines
356356 },
357357 "packages/core/execution": {
358358 "name": "@executor-js/execution",
359− "version": "1.4.30",
359+ "version": "1.4.31",
360360 "dependencies": {
361361 "@executor-js/codemode-core": "workspace:*",
362362 "@executor-js/sdk": "workspace:*",
@@ -418,7 +418,7 @@ · 55 unmodified lines
418418 },
419419 "packages/core/sdk": {
420420 "name": "@executor-js/sdk",
421− "version": "1.4.30",
421+ "version": "1.4.31",
422422 "dependencies": {
423423 "@standard-schema/spec": "^1.1.0",
424424 "effect": "catalog:",
@@ -470,7 +470,7 @@ · 45 unmodified lines
470470 },
471471 "packages/core/vite-plugin": {
472472 "name": "@executor-js/vite-plugin",
473− "version": "0.0.15",
473+ "version": "0.0.16",
474474 "dependencies": {
475475 "@executor-js/sdk": "workspace:*",
476476 "jiti": "^2.6.1",
@@ -508,7 +508,7 @@ · 31 unmodified lines
508508 },
509509 "packages/kernel/core": {
510510 "name": "@executor-js/codemode-core",
511− "version": "1.4.30",
511+ "version": "1.4.31",
512512 "dependencies": {
513513 "@babel/parser": "^7.29.2",
514514 "@standard-schema/spec": "^1.0.0",
@@ -578,7 +578,7 @@ · 63 unmodified lines
578578 },
579579 "packages/kernel/runtime-quickjs": {
580580 "name": "@executor-js/runtime-quickjs",
581− "version": "1.4.30",
581+ "version": "1.4.31",
582582 "dependencies": {
583583 "@executor-js/codemode-core": "workspace:*",
584584 "effect": "catalog:",
@@ -595,7 +595,7 @@ · 10 unmodified lines
595595 },
596596 "packages/plugins/desktop-settings": {
597597 "name": "@executor-js/plugin-desktop-settings",
598− "version": "1.4.30",
598+ "version": "1.4.31",
599599 "dependencies": {
600600 "@executor-js/sdk": "workspace:*",
601601 "react": "catalog:",
@@ -608,7 +608,7 @@ · 6 unmodified lines
608608 },
609609 "packages/plugins/example": {
610610 "name": "@executor-js/plugin-example",
611− "version": "1.4.30",
611+ "version": "1.4.31",
612612 "dependencies": {
613613 "@executor-js/sdk": "workspace:*",
614614 },
@@ -631,7 +631,7 @@ · 16 unmodified lines
631631 },
632632 "packages/plugins/file-secrets": {
633633 "name": "@executor-js/plugin-file-secrets",
634− "version": "1.4.30",
634+ "version": "1.4.31",
635635 "dependencies": {
636636 "@executor-js/sdk": "workspace:*",
637637 "effect": "catalog:",
@@ -645,7 +645,7 @@ · 7 unmodified lines
645645 },
646646 "packages/plugins/google-discovery": {
647647 "name": "@executor-js/plugin-google-discovery",
648− "version": "1.4.30",
648+ "version": "1.4.31",
649649 "dependencies": {
650650 "@executor-js/sdk": "workspace:*",
651651 "effect": "catalog:",
@@ -680,7 +680,7 @@ · 28 unmodified lines
680680 },
681681 "packages/plugins/graphql": {
682682 "name": "@executor-js/plugin-graphql",
683− "version": "1.4.30",
683+ "version": "1.4.31",
684684 "dependencies": {
685685 "@effect/platform-node": "catalog:",
686686 "@executor-js/config": "workspace:*",
@@ -718,7 +718,7 @@ · 31 unmodified lines
718718 },
719719 "packages/plugins/keychain": {
720720 "name": "@executor-js/plugin-keychain",
721− "version": "1.4.30",
721+ "version": "1.4.31",
722722 "dependencies": {
723723 "@executor-js/sdk": "workspace:*",
724724 "@napi-rs/keyring": "^1.2.0",
@@ -734,7 +734,7 @@ · 9 unmodified lines
734734 },
735735 "packages/plugins/mcp": {
736736 "name": "@executor-js/plugin-mcp",
737− "version": "1.4.30",
737+ "version": "1.4.31",
738738 "dependencies": {
739739 "@cfworker/json-schema": "^4.1.1",
740740 "@effect/platform-node": "catalog:",
@@ -773,7 +773,7 @@ · 32 unmodified lines
773773 },
774774 "packages/plugins/onepassword": {
775775 "name": "@executor-js/plugin-onepassword",
776− "version": "1.4.30",
776+ "version": "1.4.31",
777777 "dependencies": {
778778 "@1password/op-js": "^0.1.13",
779779 "@1password/sdk": "^0.4.1-beta.1",
@@ -806,7 +806,7 @@ · 26 unmodified lines
806806 },
807807 "packages/plugins/openapi": {
808808 "name": "@executor-js/plugin-openapi",
809− "version": "1.4.30",
809+ "version": "1.4.31",
810810 "dependencies": {
811811 "@effect/platform-node": "catalog:",
812812 "@executor-js/config": "workspace:*",
@@ -869,7 +869,7 @@ · 56 unmodified lines
869869 },
870870 "packages/react": {
871871 "name": "@executor-js/react",
872− "version": "1.4.18",
872+ "version": "1.4.19",
873873 "dependencies": {
874874 "@base-ui/react": "^1.3.0",
875875 "@effect/atom-react": "catalog:",
examples/all-plugins/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/example-all-plugins",
3− "version": "0.0.16",
3+ "version": "0.0.17",
44 "private": true,
55 "type": "module",
66 "scripts": {
examples/docs-sdk-quickstart/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/example-docs-sdk-quickstart",
3− "version": "0.0.1",
3+ "version": "0.0.2",
44 "private": true,
55 "type": "module",
66 "scripts": {
packages/core/api/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/api",
3− "version": "1.4.18",
3+ "version": "1.4.19",
44 "private": true,
55 "type": "module",
66 "exports": {
packages/core/cli/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/cli",
3− "version": "0.2.2",
3+ "version": "0.2.3",
44 "description": "CLI for the executor SDK — schema generation, migrations",
55 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/core/cli",
66 "bugs": {
packages/core/config/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/config",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/core/config",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/core/execution/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/execution",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/core/execution",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/core/sdk/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/sdk",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/core/sdk",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/core/vite-plugin/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/vite-plugin",
3− "version": "0.0.15",
3+ "version": "0.0.16",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/core/vite-plugin",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/kernel/core/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/codemode-core",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/kernel/core",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/kernel/runtime-quickjs/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/runtime-quickjs",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/kernel/runtime-quickjs",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/plugins/desktop-settings/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/plugin-desktop-settings",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/plugins/desktop-settings",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/plugins/example/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/plugin-example",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/plugins/example",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/plugins/file-secrets/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/plugin-file-secrets",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/plugins/file-secrets",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/plugins/google-discovery/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/plugin-google-discovery",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/plugins/google-discovery",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/plugins/graphql/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/plugin-graphql",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/plugins/graphql",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/plugins/keychain/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/plugin-keychain",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/plugins/keychain",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/plugins/mcp/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/plugin-mcp",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/plugins/mcp",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/plugins/onepassword/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/plugin-onepassword",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/plugins/onepassword",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/plugins/openapi/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/plugin-openapi",
3− "version": "1.4.30",
3+ "version": "1.4.31",
44 "homepage": "https://github.com/RhysSullivan/executor/tree/main/packages/plugins/openapi",
55 "bugs": {
66 "url": "https://github.com/RhysSullivan/executor/issues"
packages/react/package.json+1−1
@@ -1,6 +1,6 @@
11 {
22 "name": "@executor-js/react",
3− "version": "1.4.18",
3+ "version": "1.4.19",
44 "private": true,
55 "type": "module",
66 "exports": {

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t