History

Fix dev-db socket server slot leak and add a wedge watchdog (#1744)

Rhys Sullivan committed 2 months ago

Commit 1bbd47e1 parent 91062c23 files · +188 −18Browse files

Showing the changes to apps/cloud/scripts (1 of 3 files). Show all

Unified · Split

apps/cloud/scripts/dev-db.ts+131−16
@@ -15,6 +15,7 @@ import { PGlite } from "@electric-sql/pglite"; · 14 unmodified lines
15import { PGLiteSocketServer } from "@electric-sql/pglite-socket";15import { PGLiteSocketServer } from "@electric-sql/pglite-socket";
16import { drizzle } from "drizzle-orm/pglite";16import { drizzle } from "drizzle-orm/pglite";
17import { migrate } from "drizzle-orm/pglite/migrator";17import { migrate } from "drizzle-orm/pglite/migrator";
18import postgres from "postgres";
18 19
19const __dirname = dirname(fileURLToPath(import.meta.url));20const __dirname = dirname(fileURLToPath(import.meta.url));
20// Port + data dir default to the dev values but are env-overridable so a second21// Port + data dir default to the dev values but are env-overridable so a second
@@ -118,27 +119,32 @@ await migrate(drizzle(db), { migrationsFolder: MIGRATIONS_FOLDER }); · 97 unmodified lines
118// transaction or pipeline affinity they can no longer release.119// transaction or pipeline affinity they can no longer release.
119// src/db/dev-db-socket-concurrency.node.test.ts is the regression test for120// src/db/dev-db-socket-concurrency.node.test.ts is the regression test for
120// all of the above.121// all of the above.
121const server = new PGLiteSocketServer({122const makeServer = () =>
122 db,123 new PGLiteSocketServer({
123 port: PORT,124 db,
124 host: "127.0.0.1",125 port: PORT,
125 maxConnections: Number(process.env.DEV_DB_MAX_CONNECTIONS ?? 1000),126 host: "127.0.0.1",
126 // Backstop for pipeline affinity: a client that stalls mid-pipeline (Parse127 maxConnections: Number(process.env.DEV_DB_MAX_CONNECTIONS ?? 1000),
127 // sent, no Sync) with its socket still OPEN would hold the queue's handler128 // Backstop for pipeline affinity: a client that stalls mid-pipeline (Parse
128 // affinity forever and starve every other connection, since affinity only129 // sent, no Sync) with its socket still OPEN would hold the queue's handler
129 // releases on detach and detach needs close/error/idle-timeout. In ms; the130 // affinity forever and starve every other connection, since affinity only
130 // timer resets on every data event. The patch scopes the reap to connections131 // releases on detach and detach needs close/error/idle-timeout. In ms; the
131 // actually HOLDING affinity (open pipeline or transaction): an idle-at-rest132 // timer resets on every data event. The patch scopes the reap to connections
132 // connection is the normal state of a healthy postgres.js pool held by a133 // actually HOLDING affinity (open pipeline or transaction): an idle-at-rest
133 // long-lived scope (SSE), and reaping those raced live queries into134 // connection is the normal state of a healthy postgres.js pool held by a
134 // sporadic `write CONNECTION_ENDED` 500s.135 // long-lived scope (SSE), and reaping those raced live queries into
135 idleTimeout: Number(process.env.DEV_DB_IDLE_TIMEOUT_MS ?? 30_000),136 // sporadic `write CONNECTION_ENDED` 500s.
136});137 idleTimeout: Number(process.env.DEV_DB_IDLE_TIMEOUT_MS ?? 30_000),
138 });
137 139
140let server = makeServer();
138await server.start();141await server.start();
139console.log(`[dev-db] Listening on postgresql://postgres:postgres@127.0.0.1:${PORT}/postgres`);142console.log(`[dev-db] Listening on postgresql://postgres:postgres@127.0.0.1:${PORT}/postgres`);
140 143
144let stopping = false;
145
141const shutdown = async () => {146const shutdown = async () => {
147 stopping = true;
142 console.log("\n[dev-db] Shutting down");148 console.log("\n[dev-db] Shutting down");
143 await server.stop();149 await server.stop();
144 await db.close();150 await db.close();
@@ -147,3 +153,112 @@ const shutdown = async () => { · 2 unmodified lines
147 153
148process.on("SIGINT", shutdown);154process.on("SIGINT", shutdown);
149process.on("SIGTERM", shutdown);155process.on("SIGTERM", shutdown);
156
157// ---------------------------------------------------------------------------
158// Wedge watchdog
159// ---------------------------------------------------------------------------
160//
161// Twice the socket server has shipped a state machine that could stop
162// answering NEW connections while the process, the port, and PGlite all stayed
163// up (the CI e2e "cloud signIn: callback set no session (500)" cascades: every
164// in-flight query dies once, then every fresh connection's startup packet
165// times out — CONNECT_TIMEOUT — for the rest of the shard). The known paths
166// are patched with regression tests, but each recurrence so far has found a
167// new path, and a wedged front-end turns ONE infra hiccup into a failure of
168// every remaining test in the shard.
169//
170// So: probe the server the way the app does — a fresh TCP connection, real
171// startup handshake, `select 1` — and when several consecutive probes fail,
172// dump the server's internals to the boot log and swap in a fresh socket
173// server on the same PGlite instance (all state is in PGlite; the front-end is
174// stateless, so this drops only already-doomed connections). If a restart
175// doesn't restore service, exit non-zero: the boot supervisor logs the exit
176// loudly and the run fails fast with an attributable cause instead of minutes
177// of anonymous CONNECT_TIMEOUTs. The wedge itself stays visible in the
178// server-logs artifact via the [dev-db][watchdog] lines.
179const WATCHDOG_INTERVAL_MS = Number(process.env.DEV_DB_WATCHDOG_INTERVAL_MS ?? 5_000);
180// 3 consecutive failures ≈ 15s+ of hard unavailability. PGlite serves queries
181// in milliseconds; even a deep queue clears in well under one probe interval,
182// so consecutive startup failures this sustained only happen wedged.
183const WATCHDOG_FAILURES_TO_RESTART = 3;
184const WATCHDOG_MAX_RESTARTS = 3;
185
186const probe = async (): Promise<void> => {
187 const sql = postgres(`postgres://postgres:postgres@127.0.0.1:${PORT}/postgres`, {
188 max: 1,
189 idle_timeout: 0,
190 connect_timeout: 5,
191 fetch_types: false,
192 prepare: false,
193 onnotice: () => undefined,
194 });
195 try {
196 // connect_timeout only bounds the handshake; race the query too so a
197 // post-startup wedge cannot hang the watchdog itself.
198 await Promise.race([
199 sql.unsafe("select 1"),
200 sleep(10_000).then(() => {
201 throw new Error("probe query timed out after 10s");
202 }),
203 ]);
204 } finally {
205 await sql.end({ timeout: 5 }).catch(() => {});
206 }
207};
208
209const watchdog = async () => {
210 let consecutiveFailures = 0;
211 let restarts = 0;
212 for (;;) {
213 await sleep(WATCHDOG_INTERVAL_MS);
214 if (stopping) return;
215 try {
216 await probe();
217 consecutiveFailures = 0;
218 } catch (cause) {
219 consecutiveFailures += 1;
220 console.error(
221 `[dev-db][watchdog] probe failed (${consecutiveFailures}/${WATCHDOG_FAILURES_TO_RESTART}): ${String(cause)}`,
222 );
223 if (consecutiveFailures < WATCHDOG_FAILURES_TO_RESTART) continue;
224 console.error(
225 `[dev-db][watchdog] socket server wedged; stats: ${JSON.stringify(server.getStats())}`,
226 );
227 if (restarts >= WATCHDOG_MAX_RESTARTS) {
228 console.error(
229 `[dev-db][watchdog] still wedged after ${restarts} restarts — giving up so the boot supervisor reports it`,
230 );
231 process.exit(1);
232 }
233 restarts += 1;
234 consecutiveFailures = 0;
235 console.error(
236 `[dev-db][watchdog] restarting socket server (${restarts}/${WATCHDOG_MAX_RESTARTS})`,
237 );
238 // stop() itself goes through the query queue (detach rolls back open
239 // transactions), so a wedge deep enough can hang the restart too —
240 // bound it and treat that as fatal rather than hanging the watchdog.
241 const restart = async () => {
242 await server.stop();
243 server = makeServer();
244 await server.start();
245 };
246 try {
247 await Promise.race([
248 restart(),
249 sleep(15_000).then(() => {
250 throw new Error("restart timed out after 15s");
251 }),
252 ]);
253 console.error(`[dev-db][watchdog] socket server restarted`);
254 } catch (restartCause) {
255 console.error(
256 `[dev-db][watchdog] restart failed (${String(restartCause)}) — exiting so the boot supervisor reports it`,
257 );
258 process.exit(1);
259 }
260 }
261 }
262};
263
264void watchdog();

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t