Showing the changes to apps/cloud/scripts (1 of 3 files). Show all
Unified · Split
| @@ -15,6 +15,7 @@ import { PGlite } from "@electric-sql/pglite"; · 14 unmodified lines | ||
| 15 | 15 | import { PGLiteSocketServer } from "@electric-sql/pglite-socket"; |
| 16 | 16 | import { drizzle } from "drizzle-orm/pglite"; |
| 17 | 17 | import { migrate } from "drizzle-orm/pglite/migrator"; |
| 18 | +import postgres from "postgres"; | |
| 18 | 19 | |
| 19 | 20 | const __dirname = dirname(fileURLToPath(import.meta.url)); |
| 20 | 21 | // Port + data dir default to the dev values but are env-overridable so a second |
| @@ -118,27 +119,32 @@ await migrate(drizzle(db), { migrationsFolder: MIGRATIONS_FOLDER }); · 97 unmodified lines | ||
| 118 | 119 | // transaction or pipeline affinity they can no longer release. |
| 119 | 120 | // src/db/dev-db-socket-concurrency.node.test.ts is the regression test for |
| 120 | 121 | // all of the above. |
| 121 | −const server = new PGLiteSocketServer({ | |
| 122 | − db, | |
| 123 | − port: PORT, | |
| 124 | − host: "127.0.0.1", | |
| 125 | − maxConnections: Number(process.env.DEV_DB_MAX_CONNECTIONS ?? 1000), | |
| 126 | − // Backstop for pipeline affinity: a client that stalls mid-pipeline (Parse | |
| 127 | − // sent, no Sync) with its socket still OPEN would hold the queue's handler | |
| 128 | − // affinity forever and starve every other connection, since affinity only | |
| 129 | − // releases on detach and detach needs close/error/idle-timeout. In ms; the | |
| 130 | − // timer resets on every data event. The patch scopes the reap to connections | |
| 131 | − // actually HOLDING affinity (open pipeline or transaction): an idle-at-rest | |
| 132 | − // connection is the normal state of a healthy postgres.js pool held by a | |
| 133 | − // long-lived scope (SSE), and reaping those raced live queries into | |
| 134 | − // sporadic `write CONNECTION_ENDED` 500s. | |
| 135 | − idleTimeout: Number(process.env.DEV_DB_IDLE_TIMEOUT_MS ?? 30_000), | |
| 136 | −}); | |
| 122 | +const makeServer = () => | |
| 123 | + new PGLiteSocketServer({ | |
| 124 | + db, | |
| 125 | + port: PORT, | |
| 126 | + host: "127.0.0.1", | |
| 127 | + maxConnections: Number(process.env.DEV_DB_MAX_CONNECTIONS ?? 1000), | |
| 128 | + // Backstop for pipeline affinity: a client that stalls mid-pipeline (Parse | |
| 129 | + // sent, no Sync) with its socket still OPEN would hold the queue's handler | |
| 130 | + // affinity forever and starve every other connection, since affinity only | |
| 131 | + // releases on detach and detach needs close/error/idle-timeout. In ms; the | |
| 132 | + // timer resets on every data event. The patch scopes the reap to connections | |
| 133 | + // actually HOLDING affinity (open pipeline or transaction): an idle-at-rest | |
| 134 | + // connection is the normal state of a healthy postgres.js pool held by a | |
| 135 | + // long-lived scope (SSE), and reaping those raced live queries into | |
| 136 | + // sporadic `write CONNECTION_ENDED` 500s. | |
| 137 | + idleTimeout: Number(process.env.DEV_DB_IDLE_TIMEOUT_MS ?? 30_000), | |
| 138 | + }); | |
| 137 | 139 | |
| 140 | +let server = makeServer(); | |
| 138 | 141 | await server.start(); |
| 139 | 142 | console.log(`[dev-db] Listening on postgresql://postgres:postgres@127.0.0.1:${PORT}/postgres`); |
| 140 | 143 | |
| 144 | +let stopping = false; | |
| 145 | + | |
| 141 | 146 | const shutdown = async () => { |
| 147 | + stopping = true; | |
| 142 | 148 | console.log("\n[dev-db] Shutting down"); |
| 143 | 149 | await server.stop(); |
| 144 | 150 | await db.close(); |
| @@ -147,3 +153,112 @@ const shutdown = async () => { · 2 unmodified lines | ||
| 147 | 153 | |
| 148 | 154 | process.on("SIGINT", shutdown); |
| 149 | 155 | process.on("SIGTERM", shutdown); |
| 156 | + | |
| 157 | +// --------------------------------------------------------------------------- | |
| 158 | +// Wedge watchdog | |
| 159 | +// --------------------------------------------------------------------------- | |
| 160 | +// | |
| 161 | +// Twice the socket server has shipped a state machine that could stop | |
| 162 | +// answering NEW connections while the process, the port, and PGlite all stayed | |
| 163 | +// up (the CI e2e "cloud signIn: callback set no session (500)" cascades: every | |
| 164 | +// in-flight query dies once, then every fresh connection's startup packet | |
| 165 | +// times out — CONNECT_TIMEOUT — for the rest of the shard). The known paths | |
| 166 | +// are patched with regression tests, but each recurrence so far has found a | |
| 167 | +// new path, and a wedged front-end turns ONE infra hiccup into a failure of | |
| 168 | +// every remaining test in the shard. | |
| 169 | +// | |
| 170 | +// So: probe the server the way the app does — a fresh TCP connection, real | |
| 171 | +// startup handshake, `select 1` — and when several consecutive probes fail, | |
| 172 | +// dump the server's internals to the boot log and swap in a fresh socket | |
| 173 | +// server on the same PGlite instance (all state is in PGlite; the front-end is | |
| 174 | +// stateless, so this drops only already-doomed connections). If a restart | |
| 175 | +// doesn't restore service, exit non-zero: the boot supervisor logs the exit | |
| 176 | +// loudly and the run fails fast with an attributable cause instead of minutes | |
| 177 | +// of anonymous CONNECT_TIMEOUTs. The wedge itself stays visible in the | |
| 178 | +// server-logs artifact via the [dev-db][watchdog] lines. | |
| 179 | +const WATCHDOG_INTERVAL_MS = Number(process.env.DEV_DB_WATCHDOG_INTERVAL_MS ?? 5_000); | |
| 180 | +// 3 consecutive failures ≈ 15s+ of hard unavailability. PGlite serves queries | |
| 181 | +// in milliseconds; even a deep queue clears in well under one probe interval, | |
| 182 | +// so consecutive startup failures this sustained only happen wedged. | |
| 183 | +const WATCHDOG_FAILURES_TO_RESTART = 3; | |
| 184 | +const WATCHDOG_MAX_RESTARTS = 3; | |
| 185 | + | |
| 186 | +const probe = async (): Promise<void> => { | |
| 187 | + const sql = postgres(`postgres://postgres:postgres@127.0.0.1:${PORT}/postgres`, { | |
| 188 | + max: 1, | |
| 189 | + idle_timeout: 0, | |
| 190 | + connect_timeout: 5, | |
| 191 | + fetch_types: false, | |
| 192 | + prepare: false, | |
| 193 | + onnotice: () => undefined, | |
| 194 | + }); | |
| 195 | + try { | |
| 196 | + // connect_timeout only bounds the handshake; race the query too so a | |
| 197 | + // post-startup wedge cannot hang the watchdog itself. | |
| 198 | + await Promise.race([ | |
| 199 | + sql.unsafe("select 1"), | |
| 200 | + sleep(10_000).then(() => { | |
| 201 | + throw new Error("probe query timed out after 10s"); | |
| 202 | + }), | |
| 203 | + ]); | |
| 204 | + } finally { | |
| 205 | + await sql.end({ timeout: 5 }).catch(() => {}); | |
| 206 | + } | |
| 207 | +}; | |
| 208 | + | |
| 209 | +const watchdog = async () => { | |
| 210 | + let consecutiveFailures = 0; | |
| 211 | + let restarts = 0; | |
| 212 | + for (;;) { | |
| 213 | + await sleep(WATCHDOG_INTERVAL_MS); | |
| 214 | + if (stopping) return; | |
| 215 | + try { | |
| 216 | + await probe(); | |
| 217 | + consecutiveFailures = 0; | |
| 218 | + } catch (cause) { | |
| 219 | + consecutiveFailures += 1; | |
| 220 | + console.error( | |
| 221 | + `[dev-db][watchdog] probe failed (${consecutiveFailures}/${WATCHDOG_FAILURES_TO_RESTART}): ${String(cause)}`, | |
| 222 | + ); | |
| 223 | + if (consecutiveFailures < WATCHDOG_FAILURES_TO_RESTART) continue; | |
| 224 | + console.error( | |
| 225 | + `[dev-db][watchdog] socket server wedged; stats: ${JSON.stringify(server.getStats())}`, | |
| 226 | + ); | |
| 227 | + if (restarts >= WATCHDOG_MAX_RESTARTS) { | |
| 228 | + console.error( | |
| 229 | + `[dev-db][watchdog] still wedged after ${restarts} restarts — giving up so the boot supervisor reports it`, | |
| 230 | + ); | |
| 231 | + process.exit(1); | |
| 232 | + } | |
| 233 | + restarts += 1; | |
| 234 | + consecutiveFailures = 0; | |
| 235 | + console.error( | |
| 236 | + `[dev-db][watchdog] restarting socket server (${restarts}/${WATCHDOG_MAX_RESTARTS})`, | |
| 237 | + ); | |
| 238 | + // stop() itself goes through the query queue (detach rolls back open | |
| 239 | + // transactions), so a wedge deep enough can hang the restart too — | |
| 240 | + // bound it and treat that as fatal rather than hanging the watchdog. | |
| 241 | + const restart = async () => { | |
| 242 | + await server.stop(); | |
| 243 | + server = makeServer(); | |
| 244 | + await server.start(); | |
| 245 | + }; | |
| 246 | + try { | |
| 247 | + await Promise.race([ | |
| 248 | + restart(), | |
| 249 | + sleep(15_000).then(() => { | |
| 250 | + throw new Error("restart timed out after 15s"); | |
| 251 | + }), | |
| 252 | + ]); | |
| 253 | + console.error(`[dev-db][watchdog] socket server restarted`); | |
| 254 | + } catch (restartCause) { | |
| 255 | + console.error( | |
| 256 | + `[dev-db][watchdog] restart failed (${String(restartCause)}) — exiting so the boot supervisor reports it`, | |
| 257 | + ); | |
| 258 | + process.exit(1); | |
| 259 | + } | |
| 260 | + } | |
| 261 | + } | |
| 262 | +}; | |
| 263 | + | |
| 264 | +void watchdog(); | |