History

Fix dev-db socket server slot leak and add a wedge watchdog (#1744)

Rhys Sullivan committed 2 months ago

Commit 1bbd47e1 parent 91062c23 files · +188 −18Browse files

Showing the changes to apps/cloud/scripts (1 of 3 files). Show all

Unified · Split

apps/cloud/scripts/dev-db.ts+131−16
@@ -15,6 +15,7 @@ import { PGlite } from "@electric-sql/pglite"; · 14 unmodified lines
1515 import { PGLiteSocketServer } from "@electric-sql/pglite-socket";
1616 import { drizzle } from "drizzle-orm/pglite";
1717 import { migrate } from "drizzle-orm/pglite/migrator";
18+import postgres from "postgres";
1819
1920 const __dirname = dirname(fileURLToPath(import.meta.url));
2021 // Port + data dir default to the dev values but are env-overridable so a second
@@ -118,27 +119,32 @@ await migrate(drizzle(db), { migrationsFolder: MIGRATIONS_FOLDER }); · 97 unmodified lines
118119 // transaction or pipeline affinity they can no longer release.
119120 // src/db/dev-db-socket-concurrency.node.test.ts is the regression test for
120121 // all of the above.
121−const server = new PGLiteSocketServer({
122− db,
123− port: PORT,
124− host: "127.0.0.1",
125− maxConnections: Number(process.env.DEV_DB_MAX_CONNECTIONS ?? 1000),
126− // Backstop for pipeline affinity: a client that stalls mid-pipeline (Parse
127− // sent, no Sync) with its socket still OPEN would hold the queue's handler
128− // affinity forever and starve every other connection, since affinity only
129− // releases on detach and detach needs close/error/idle-timeout. In ms; the
130− // timer resets on every data event. The patch scopes the reap to connections
131− // actually HOLDING affinity (open pipeline or transaction): an idle-at-rest
132− // connection is the normal state of a healthy postgres.js pool held by a
133− // long-lived scope (SSE), and reaping those raced live queries into
134− // sporadic `write CONNECTION_ENDED` 500s.
135− idleTimeout: Number(process.env.DEV_DB_IDLE_TIMEOUT_MS ?? 30_000),
136−});
122+const makeServer = () =>
123+ new PGLiteSocketServer({
124+ db,
125+ port: PORT,
126+ host: "127.0.0.1",
127+ maxConnections: Number(process.env.DEV_DB_MAX_CONNECTIONS ?? 1000),
128+ // Backstop for pipeline affinity: a client that stalls mid-pipeline (Parse
129+ // sent, no Sync) with its socket still OPEN would hold the queue's handler
130+ // affinity forever and starve every other connection, since affinity only
131+ // releases on detach and detach needs close/error/idle-timeout. In ms; the
132+ // timer resets on every data event. The patch scopes the reap to connections
133+ // actually HOLDING affinity (open pipeline or transaction): an idle-at-rest
134+ // connection is the normal state of a healthy postgres.js pool held by a
135+ // long-lived scope (SSE), and reaping those raced live queries into
136+ // sporadic `write CONNECTION_ENDED` 500s.
137+ idleTimeout: Number(process.env.DEV_DB_IDLE_TIMEOUT_MS ?? 30_000),
138+ });
137139
140+let server = makeServer();
138141 await server.start();
139142 console.log(`[dev-db] Listening on postgresql://postgres:postgres@127.0.0.1:${PORT}/postgres`);
140143
144+let stopping = false;
145+
141146 const shutdown = async () => {
147+ stopping = true;
142148 console.log("\n[dev-db] Shutting down");
143149 await server.stop();
144150 await db.close();
@@ -147,3 +153,112 @@ const shutdown = async () => { · 2 unmodified lines
147153
148154 process.on("SIGINT", shutdown);
149155 process.on("SIGTERM", shutdown);
156+
157+// ---------------------------------------------------------------------------
158+// Wedge watchdog
159+// ---------------------------------------------------------------------------
160+//
161+// Twice the socket server has shipped a state machine that could stop
162+// answering NEW connections while the process, the port, and PGlite all stayed
163+// up (the CI e2e "cloud signIn: callback set no session (500)" cascades: every
164+// in-flight query dies once, then every fresh connection's startup packet
165+// times out — CONNECT_TIMEOUT — for the rest of the shard). The known paths
166+// are patched with regression tests, but each recurrence so far has found a
167+// new path, and a wedged front-end turns ONE infra hiccup into a failure of
168+// every remaining test in the shard.
169+//
170+// So: probe the server the way the app does — a fresh TCP connection, real
171+// startup handshake, `select 1` — and when several consecutive probes fail,
172+// dump the server's internals to the boot log and swap in a fresh socket
173+// server on the same PGlite instance (all state is in PGlite; the front-end is
174+// stateless, so this drops only already-doomed connections). If a restart
175+// doesn't restore service, exit non-zero: the boot supervisor logs the exit
176+// loudly and the run fails fast with an attributable cause instead of minutes
177+// of anonymous CONNECT_TIMEOUTs. The wedge itself stays visible in the
178+// server-logs artifact via the [dev-db][watchdog] lines.
179+const WATCHDOG_INTERVAL_MS = Number(process.env.DEV_DB_WATCHDOG_INTERVAL_MS ?? 5_000);
180+// 3 consecutive failures ≈ 15s+ of hard unavailability. PGlite serves queries
181+// in milliseconds; even a deep queue clears in well under one probe interval,
182+// so consecutive startup failures this sustained only happen wedged.
183+const WATCHDOG_FAILURES_TO_RESTART = 3;
184+const WATCHDOG_MAX_RESTARTS = 3;
185+
186+const probe = async (): Promise<void> => {
187+ const sql = postgres(`postgres://postgres:postgres@127.0.0.1:${PORT}/postgres`, {
188+ max: 1,
189+ idle_timeout: 0,
190+ connect_timeout: 5,
191+ fetch_types: false,
192+ prepare: false,
193+ onnotice: () => undefined,
194+ });
195+ try {
196+ // connect_timeout only bounds the handshake; race the query too so a
197+ // post-startup wedge cannot hang the watchdog itself.
198+ await Promise.race([
199+ sql.unsafe("select 1"),
200+ sleep(10_000).then(() => {
201+ throw new Error("probe query timed out after 10s");
202+ }),
203+ ]);
204+ } finally {
205+ await sql.end({ timeout: 5 }).catch(() => {});
206+ }
207+};
208+
209+const watchdog = async () => {
210+ let consecutiveFailures = 0;
211+ let restarts = 0;
212+ for (;;) {
213+ await sleep(WATCHDOG_INTERVAL_MS);
214+ if (stopping) return;
215+ try {
216+ await probe();
217+ consecutiveFailures = 0;
218+ } catch (cause) {
219+ consecutiveFailures += 1;
220+ console.error(
221+ `[dev-db][watchdog] probe failed (${consecutiveFailures}/${WATCHDOG_FAILURES_TO_RESTART}): ${String(cause)}`,
222+ );
223+ if (consecutiveFailures < WATCHDOG_FAILURES_TO_RESTART) continue;
224+ console.error(
225+ `[dev-db][watchdog] socket server wedged; stats: ${JSON.stringify(server.getStats())}`,
226+ );
227+ if (restarts >= WATCHDOG_MAX_RESTARTS) {
228+ console.error(
229+ `[dev-db][watchdog] still wedged after ${restarts} restarts — giving up so the boot supervisor reports it`,
230+ );
231+ process.exit(1);
232+ }
233+ restarts += 1;
234+ consecutiveFailures = 0;
235+ console.error(
236+ `[dev-db][watchdog] restarting socket server (${restarts}/${WATCHDOG_MAX_RESTARTS})`,
237+ );
238+ // stop() itself goes through the query queue (detach rolls back open
239+ // transactions), so a wedge deep enough can hang the restart too —
240+ // bound it and treat that as fatal rather than hanging the watchdog.
241+ const restart = async () => {
242+ await server.stop();
243+ server = makeServer();
244+ await server.start();
245+ };
246+ try {
247+ await Promise.race([
248+ restart(),
249+ sleep(15_000).then(() => {
250+ throw new Error("restart timed out after 15s");
251+ }),
252+ ]);
253+ console.error(`[dev-db][watchdog] socket server restarted`);
254+ } catch (restartCause) {
255+ console.error(
256+ `[dev-db][watchdog] restart failed (${String(restartCause)}) — exiting so the boot supervisor reports it`,
257+ );
258+ process.exit(1);
259+ }
260+ }
261+ }
262+};
263+
264+void watchdog();

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t