Showing the changes to apps/cloud/scripts (1 of 3 files). Show all
Unified · Split
| @@ -15,6 +15,7 @@ import { PGlite } from "@electric-sql/pglite"; · 14 unmodified lines | |||
| 15 | import { PGLiteSocketServer } from "@electric-sql/pglite-socket"; | 15 | import { PGLiteSocketServer } from "@electric-sql/pglite-socket"; |
| 16 | import { drizzle } from "drizzle-orm/pglite"; | 16 | import { drizzle } from "drizzle-orm/pglite"; |
| 17 | import { migrate } from "drizzle-orm/pglite/migrator"; | 17 | import { migrate } from "drizzle-orm/pglite/migrator"; |
| 18 | import postgres from "postgres"; | ||
| 18 | 19 | ||
| 19 | const __dirname = dirname(fileURLToPath(import.meta.url)); | 20 | const __dirname = dirname(fileURLToPath(import.meta.url)); |
| 20 | // Port + data dir default to the dev values but are env-overridable so a second | 21 | // Port + data dir default to the dev values but are env-overridable so a second |
| @@ -118,27 +119,32 @@ await migrate(drizzle(db), { migrationsFolder: MIGRATIONS_FOLDER }); · 97 unmodified lines | |||
| 118 | // transaction or pipeline affinity they can no longer release. | 119 | // transaction or pipeline affinity they can no longer release. |
| 119 | // src/db/dev-db-socket-concurrency.node.test.ts is the regression test for | 120 | // src/db/dev-db-socket-concurrency.node.test.ts is the regression test for |
| 120 | // all of the above. | 121 | // all of the above. |
| 121 | const server = new PGLiteSocketServer({ | 122 | const makeServer = () => |
| 122 | db, | 123 | new PGLiteSocketServer({ |
| 123 | port: PORT, | 124 | db, |
| 124 | host: "127.0.0.1", | 125 | port: PORT, |
| 125 | maxConnections: Number(process.env.DEV_DB_MAX_CONNECTIONS ?? 1000), | 126 | host: "127.0.0.1", |
| 126 | // Backstop for pipeline affinity: a client that stalls mid-pipeline (Parse | 127 | maxConnections: Number(process.env.DEV_DB_MAX_CONNECTIONS ?? 1000), |
| 127 | // sent, no Sync) with its socket still OPEN would hold the queue's handler | 128 | // Backstop for pipeline affinity: a client that stalls mid-pipeline (Parse |
| 128 | // affinity forever and starve every other connection, since affinity only | 129 | // sent, no Sync) with its socket still OPEN would hold the queue's handler |
| 129 | // releases on detach and detach needs close/error/idle-timeout. In ms; the | 130 | // affinity forever and starve every other connection, since affinity only |
| 130 | // timer resets on every data event. The patch scopes the reap to connections | 131 | // releases on detach and detach needs close/error/idle-timeout. In ms; the |
| 131 | // actually HOLDING affinity (open pipeline or transaction): an idle-at-rest | 132 | // timer resets on every data event. The patch scopes the reap to connections |
| 132 | // connection is the normal state of a healthy postgres.js pool held by a | 133 | // actually HOLDING affinity (open pipeline or transaction): an idle-at-rest |
| 133 | // long-lived scope (SSE), and reaping those raced live queries into | 134 | // connection is the normal state of a healthy postgres.js pool held by a |
| 134 | // sporadic `write CONNECTION_ENDED` 500s. | 135 | // long-lived scope (SSE), and reaping those raced live queries into |
| 135 | idleTimeout: Number(process.env.DEV_DB_IDLE_TIMEOUT_MS ?? 30_000), | 136 | // sporadic `write CONNECTION_ENDED` 500s. |
| 136 | }); | 137 | idleTimeout: Number(process.env.DEV_DB_IDLE_TIMEOUT_MS ?? 30_000), |
| 138 | }); | ||
| 137 | 139 | ||
| 140 | let server = makeServer(); | ||
| 138 | await server.start(); | 141 | await server.start(); |
| 139 | console.log(`[dev-db] Listening on postgresql://postgres:postgres@127.0.0.1:${PORT}/postgres`); | 142 | console.log(`[dev-db] Listening on postgresql://postgres:postgres@127.0.0.1:${PORT}/postgres`); |
| 140 | 143 | ||
| 144 | let stopping = false; | ||
| 145 | |||
| 141 | const shutdown = async () => { | 146 | const shutdown = async () => { |
| 147 | stopping = true; | ||
| 142 | console.log("\n[dev-db] Shutting down"); | 148 | console.log("\n[dev-db] Shutting down"); |
| 143 | await server.stop(); | 149 | await server.stop(); |
| 144 | await db.close(); | 150 | await db.close(); |
| @@ -147,3 +153,112 @@ const shutdown = async () => { · 2 unmodified lines | |||
| 147 | 153 | ||
| 148 | process.on("SIGINT", shutdown); | 154 | process.on("SIGINT", shutdown); |
| 149 | process.on("SIGTERM", shutdown); | 155 | process.on("SIGTERM", shutdown); |
| 156 | |||
| 157 | // --------------------------------------------------------------------------- | ||
| 158 | // Wedge watchdog | ||
| 159 | // --------------------------------------------------------------------------- | ||
| 160 | // | ||
| 161 | // Twice the socket server has shipped a state machine that could stop | ||
| 162 | // answering NEW connections while the process, the port, and PGlite all stayed | ||
| 163 | // up (the CI e2e "cloud signIn: callback set no session (500)" cascades: every | ||
| 164 | // in-flight query dies once, then every fresh connection's startup packet | ||
| 165 | // times out — CONNECT_TIMEOUT — for the rest of the shard). The known paths | ||
| 166 | // are patched with regression tests, but each recurrence so far has found a | ||
| 167 | // new path, and a wedged front-end turns ONE infra hiccup into a failure of | ||
| 168 | // every remaining test in the shard. | ||
| 169 | // | ||
| 170 | // So: probe the server the way the app does — a fresh TCP connection, real | ||
| 171 | // startup handshake, `select 1` — and when several consecutive probes fail, | ||
| 172 | // dump the server's internals to the boot log and swap in a fresh socket | ||
| 173 | // server on the same PGlite instance (all state is in PGlite; the front-end is | ||
| 174 | // stateless, so this drops only already-doomed connections). If a restart | ||
| 175 | // doesn't restore service, exit non-zero: the boot supervisor logs the exit | ||
| 176 | // loudly and the run fails fast with an attributable cause instead of minutes | ||
| 177 | // of anonymous CONNECT_TIMEOUTs. The wedge itself stays visible in the | ||
| 178 | // server-logs artifact via the [dev-db][watchdog] lines. | ||
| 179 | const WATCHDOG_INTERVAL_MS = Number(process.env.DEV_DB_WATCHDOG_INTERVAL_MS ?? 5_000); | ||
| 180 | // 3 consecutive failures ≈ 15s+ of hard unavailability. PGlite serves queries | ||
| 181 | // in milliseconds; even a deep queue clears in well under one probe interval, | ||
| 182 | // so consecutive startup failures this sustained only happen wedged. | ||
| 183 | const WATCHDOG_FAILURES_TO_RESTART = 3; | ||
| 184 | const WATCHDOG_MAX_RESTARTS = 3; | ||
| 185 | |||
| 186 | const probe = async (): Promise<void> => { | ||
| 187 | const sql = postgres(`postgres://postgres:postgres@127.0.0.1:${PORT}/postgres`, { | ||
| 188 | max: 1, | ||
| 189 | idle_timeout: 0, | ||
| 190 | connect_timeout: 5, | ||
| 191 | fetch_types: false, | ||
| 192 | prepare: false, | ||
| 193 | onnotice: () => undefined, | ||
| 194 | }); | ||
| 195 | try { | ||
| 196 | // connect_timeout only bounds the handshake; race the query too so a | ||
| 197 | // post-startup wedge cannot hang the watchdog itself. | ||
| 198 | await Promise.race([ | ||
| 199 | sql.unsafe("select 1"), | ||
| 200 | sleep(10_000).then(() => { | ||
| 201 | throw new Error("probe query timed out after 10s"); | ||
| 202 | }), | ||
| 203 | ]); | ||
| 204 | } finally { | ||
| 205 | await sql.end({ timeout: 5 }).catch(() => {}); | ||
| 206 | } | ||
| 207 | }; | ||
| 208 | |||
| 209 | const watchdog = async () => { | ||
| 210 | let consecutiveFailures = 0; | ||
| 211 | let restarts = 0; | ||
| 212 | for (;;) { | ||
| 213 | await sleep(WATCHDOG_INTERVAL_MS); | ||
| 214 | if (stopping) return; | ||
| 215 | try { | ||
| 216 | await probe(); | ||
| 217 | consecutiveFailures = 0; | ||
| 218 | } catch (cause) { | ||
| 219 | consecutiveFailures += 1; | ||
| 220 | console.error( | ||
| 221 | `[dev-db][watchdog] probe failed (${consecutiveFailures}/${WATCHDOG_FAILURES_TO_RESTART}): ${String(cause)}`, | ||
| 222 | ); | ||
| 223 | if (consecutiveFailures < WATCHDOG_FAILURES_TO_RESTART) continue; | ||
| 224 | console.error( | ||
| 225 | `[dev-db][watchdog] socket server wedged; stats: ${JSON.stringify(server.getStats())}`, | ||
| 226 | ); | ||
| 227 | if (restarts >= WATCHDOG_MAX_RESTARTS) { | ||
| 228 | console.error( | ||
| 229 | `[dev-db][watchdog] still wedged after ${restarts} restarts — giving up so the boot supervisor reports it`, | ||
| 230 | ); | ||
| 231 | process.exit(1); | ||
| 232 | } | ||
| 233 | restarts += 1; | ||
| 234 | consecutiveFailures = 0; | ||
| 235 | console.error( | ||
| 236 | `[dev-db][watchdog] restarting socket server (${restarts}/${WATCHDOG_MAX_RESTARTS})`, | ||
| 237 | ); | ||
| 238 | // stop() itself goes through the query queue (detach rolls back open | ||
| 239 | // transactions), so a wedge deep enough can hang the restart too — | ||
| 240 | // bound it and treat that as fatal rather than hanging the watchdog. | ||
| 241 | const restart = async () => { | ||
| 242 | await server.stop(); | ||
| 243 | server = makeServer(); | ||
| 244 | await server.start(); | ||
| 245 | }; | ||
| 246 | try { | ||
| 247 | await Promise.race([ | ||
| 248 | restart(), | ||
| 249 | sleep(15_000).then(() => { | ||
| 250 | throw new Error("restart timed out after 15s"); | ||
| 251 | }), | ||
| 252 | ]); | ||
| 253 | console.error(`[dev-db][watchdog] socket server restarted`); | ||
| 254 | } catch (restartCause) { | ||
| 255 | console.error( | ||
| 256 | `[dev-db][watchdog] restart failed (${String(restartCause)}) — exiting so the boot supervisor reports it`, | ||
| 257 | ); | ||
| 258 | process.exit(1); | ||
| 259 | } | ||
| 260 | } | ||
| 261 | } | ||
| 262 | }; | ||
| 263 | |||
| 264 | void watchdog(); | ||