History

fix(local): survive Windows libSQL handle linger during v1->v2 migration (#1209)

The v1->v2 data migration ran fsync/rename/remove on libSQL SQLite files whose native OS handle lingers past close() on Windows, crashing the daemon at boot with a fatal "Unknown error". POSIX never hits this, so it only reproduced on Windows (the v1.5.23 desktop/CLI startup regression). - fsyncFileIfExists opens read-write, since Windows FlushFileBuffers needs a writable handle (a read-only fd throws EPERM), and treats fsync as best-effort durability hardening. - remove of the source/staging file sets now retries EBUSY/EPERM the same way rename already did. - rename/remove retries force a GC pass first so libSQL's native finalizer releases the handle before the next attempt. Verified on a Windows VM: a seeded v1 database now migrates to v2 and the daemon announces ready.

Rhys Sullivan committed 4 months ago

Commit ffa4f701 parent b1d73bf2 files · +62 −14Browse files

Showing the changes to apps (1 of 2 files). Show all

Unified · Split

apps/local/src/db/v1-v2-migration.ts+57−14
@@ -1003,10 +1003,19 @@ const insertPlan = async ( · 1002 unmodified lines
1003 }1003 }
1004};1004};
1005 1005
1006// libSQL releases its OS file handle in a native finalizer, which on Windows
1007// only runs on garbage collection and so lags client.close(). Forcing a GC
1008// runs the finalizer, releasing the handle so a follow-up rename/rm succeeds.
1009// No-op outside the Bun runtime (e.g. vitest on Node), where this is unneeded.
1010const nudgeNativeHandleRelease = (): void => {
1011 const bun = (globalThis as { Bun?: { gc?: (sync: boolean) => void } }).Bun;
1012 bun?.gc?.(true);
1013};
1014
1006// Windows reports EBUSY/EPERM on rename for a short window after a SQLite1015// Windows reports EBUSY/EPERM on rename for a short window after a SQLite
1007// handle closes (handle release lags the close call, and antivirus scanners1016// handle closes (handle release lags the close call, and antivirus scanners
1008// briefly lock the file). POSIX renames never hit this — retry with a short1017// briefly lock the file). POSIX renames never hit this — retry with a short
1009// backoff instead of failing the whole migration.1018// backoff, forcing handle release first, instead of failing the whole migration.
1010const renameWithRetry = async (source: string, target: string): Promise<void> => {1019const renameWithRetry = async (source: string, target: string): Promise<void> => {
1011 // ~8s total. 1.9s was not enough on Windows: libSQL's native handle and1020 // ~8s total. 1.9s was not enough on Windows: libSQL's native handle and
1012 // antivirus scans hold the freshly-written db past close() (observed as an1021 // antivirus scans hold the freshly-written db past close() (observed as an
@@ -1019,16 +1028,51 @@ const renameWithRetry = async (source: string, target: string): Promise<void> => · 6 unmodified lines
1019 } catch (cause) {1028 } catch (cause) {
1020 const code = (cause as NodeJS.ErrnoException).code;1029 const code = (cause as NodeJS.ErrnoException).code;
1021 if ((code !== "EBUSY" && code !== "EPERM") || attempt >= delaysMs.length) throw cause;1030 if ((code !== "EBUSY" && code !== "EPERM") || attempt >= delaysMs.length) throw cause;
1031 nudgeNativeHandleRelease();
1032 await new Promise((resolveDelay) => setTimeout(resolveDelay, delaysMs[attempt]));
1033 }
1034 }
1035};
1036
1037// Same lingering-handle window as renameWithRetry: on Windows a libSQL file's
1038// native handle (and antivirus locks) can briefly outlive close(), so removing
1039// a just-closed source/staging db throws EBUSY/EPERM. POSIX never hits this.
1040const rmWithRetry = async (path: string): Promise<void> => {
1041 const delaysMs = [50, 100, 250, 500, 1000, 2000, 4000];
1042 for (let attempt = 0; ; attempt++) {
1043 try {
1044 fs.rmSync(path, { force: true });
1045 return;
1046 } catch (cause) {
1047 const code = (cause as NodeJS.ErrnoException).code;
1048 if ((code !== "EBUSY" && code !== "EPERM") || attempt >= delaysMs.length) throw cause;
1049 nudgeNativeHandleRelease();
1022 await new Promise((resolveDelay) => setTimeout(resolveDelay, delaysMs[attempt]));1050 await new Promise((resolveDelay) => setTimeout(resolveDelay, delaysMs[attempt]));
1023 }1051 }
1024 }1052 }
1025};1053};
1026 1054
1055const removeSqliteFileSetWithRetry = async (path: string): Promise<void> => {
1056 for (const suffix of fileSetSuffixes) await rmWithRetry(`${path}${suffix}`);
1057};
1058
1027const fsyncFileIfExists = (path: string): void => {1059const fsyncFileIfExists = (path: string): void => {
1028 if (!fs.existsSync(path)) return;1060 if (!fs.existsSync(path)) return;
1029 const fd = fs.openSync(path, "r");1061 // Windows FlushFileBuffers requires a writable handle, so fsync on a
1062 // read-only ("r") fd throws EPERM and would crash the v1->v2 migration at
1063 // boot. Open read-write for the flush, and treat fsync as best-effort
1064 // durability hardening (the rename/copy already wrote the bytes) so a
1065 // platform or filesystem that refuses the flush does not fail the migration.
1066 let fd: number;
1067 try {
1068 fd = fs.openSync(path, "r+");
1069 } catch {
1070 return;
1071 }
1030 try {1072 try {
1031 fs.fsyncSync(fd);1073 fs.fsyncSync(fd);
1074 } catch {
1075 // best-effort: some platforms/filesystems refuse fsync on certain handles
1032 } finally {1076 } finally {
1033 fs.closeSync(fd);1077 fs.closeSync(fd);
1034 }1078 }
@@ -1060,7 +1104,7 @@ const moveSqliteFileSet = async (source: string, target: string): Promise<void> · 25 unmodified lines
1060 await renameWithRetry(`${source}${suffix}`, `${target}${suffix}`);1104 await renameWithRetry(`${source}${suffix}`, `${target}${suffix}`);
1061 fsyncDirectory(dirname(target));1105 fsyncDirectory(dirname(target));
1062 } else {1106 } else {
1063 fs.rmSync(`${target}${suffix}`, { force: true });1107 await rmWithRetry(`${target}${suffix}`);
1064 }1108 }
1065 }1109 }
1066};1110};
@@ -1071,7 +1115,7 @@ const moveExistingSqliteFileSet = async (source: string, target: string): Promis · 4 unmodified lines
1071 if (!fs.existsSync(from)) continue;1115 if (!fs.existsSync(from)) continue;
1072 const to = `${target}${suffix}`;1116 const to = `${target}${suffix}`;
1073 if (fs.existsSync(to)) {1117 if (fs.existsSync(to)) {
1074 fs.rmSync(from, { force: true });1118 await rmWithRetry(from);
1075 } else {1119 } else {
1076 await renameWithRetry(from, to);1120 await renameWithRetry(from, to);
1077 fsyncDirectory(dirname(to));1121 fsyncDirectory(dirname(to));
@@ -1413,8 +1457,8 @@ const completeJournaledFlip = async ( · 335 unmodified lines
1413 await pauseMigrationForTest("staging-consumed");1457 await pauseMigrationForTest("staging-consumed");
1414 }1458 }
1415 await writeMigrationJournal(journalPath, { ...journal, phase: "committed" });1459 await writeMigrationJournal(journalPath, { ...journal, phase: "committed" });
1416 removeSqliteFileSet(journal.normalizedSource);1460 await removeSqliteFileSetWithRetry(journal.normalizedSource);
1417 removeSqliteFileSet(journal.staging);1461 await removeSqliteFileSetWithRetry(journal.staging);
1418 await cleanupSecretBackups(journal);1462 await cleanupSecretBackups(journal);
1419 removeMigrationJournal(journalPath);1463 removeMigrationJournal(journalPath);
1420};1464};
@@ -1435,8 +1479,8 @@ const recoverV1V2Migration = async (sqlitePath: string): Promise<MigrationRecove · 14 unmodified lines
1435 1479
1436 const { journal } = journalRead;1480 const { journal } = journalRead;
1437 if (journal.phase === "building") {1481 if (journal.phase === "building") {
1438 removeSqliteFileSet(journal.normalizedSource);1482 await removeSqliteFileSetWithRetry(journal.normalizedSource);
1439 removeSqliteFileSet(journal.staging);1483 await removeSqliteFileSetWithRetry(journal.staging);
1440 await restoreAuthFromJournal(journal);1484 await restoreAuthFromJournal(journal);
1441 await cleanupSecretBackups(journal);1485 await cleanupSecretBackups(journal);
1442 removeMigrationJournal(journalPath);1486 removeMigrationJournal(journalPath);
@@ -1448,8 +1492,8 @@ const recoverV1V2Migration = async (sqlitePath: string): Promise<MigrationRecove · 5 unmodified lines
1448 return "recovered";1492 return "recovered";
1449 }1493 }
1450 1494
1451 removeSqliteFileSet(journal.normalizedSource);1495 await removeSqliteFileSetWithRetry(journal.normalizedSource);
1452 removeSqliteFileSet(journal.staging);1496 await removeSqliteFileSetWithRetry(journal.staging);
1453 await cleanupSecretBackups(journal);1497 await cleanupSecretBackups(journal);
1454 removeMigrationJournal(journalPath);1498 removeMigrationJournal(journalPath);
1455 return "recovered";1499 return "recovered";
@@ -1573,8 +1617,7 @@ export const migrateLocalV1ToV2IfNeeded = async ( · 117 unmodified lines
1573 // WAL/SHM sidecars before journaling the staged DB as complete; otherwise a1617 // WAL/SHM sidecars before journaling the staged DB as complete; otherwise a
1574 // crash mid-sidecar move would make staging appear present after the base DB1618 // crash mid-sidecar move would make staging appear present after the base DB
1575 // had already been installed.1619 // had already been installed.
1576 for (const suffix of ["-wal", "-shm"] as const)1620 for (const suffix of ["-wal", "-shm"] as const) await rmWithRetry(`${stagingPath}${suffix}`);
1577 fs.rmSync(`${stagingPath}${suffix}`, { force: true });
1578 fsyncDirectory(dirname(stagingPath));1621 fsyncDirectory(dirname(stagingPath));
1579 await writeMigrationJournal(journalPath, { ...journal, phase: "built" });1622 await writeMigrationJournal(journalPath, { ...journal, phase: "built" });
1580 await pauseMigrationForTest("built");1623 await pauseMigrationForTest("built");
@@ -1598,8 +1641,8 @@ export const migrateLocalV1ToV2IfNeeded = async ( · 17 unmodified lines
1598 await restoreAuthFromJournal(journal);1641 await restoreAuthFromJournal(journal);
1599 await cleanupSecretBackups(journal);1642 await cleanupSecretBackups(journal);
1600 }1643 }
1601 if (normalizedSourcePath) removeSqliteFileSet(normalizedSourcePath);1644 if (normalizedSourcePath) await removeSqliteFileSetWithRetry(normalizedSourcePath);
1602 if (stagingPath) removeSqliteFileSet(stagingPath);1645 if (stagingPath) await removeSqliteFileSetWithRetry(stagingPath);
1603 removeMigrationJournal(journalPath);1646 removeMigrationJournal(journalPath);
1604 }1647 }
1605 throw cause;1648 throw cause;

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t