History

Own the thread ceiling where the bindings are built

The cap was declared in the CLI while the constraint belongs to the seeder's statements, and the number appeared in three places. That split is the mechanism behind the last regression: a change to a statement in the library silently invalidated a bound stated in the CLI, and the mismatch only surfaced at the maximum — after the target was emptied. Move it to `MAX_SEED_THREAD_LIMIT` beside the statements that have to satisfy it, have the flag derive its bound from there, and check it in `seedDevDatabase` before anything is opened so a caller bypassing the flag fails while the target is still intact.

Theo Browne committed 3 months ago

Commit d3ee18c1 parent 33e4c763 files · +66 −19Browse files

Showing the changes to scripts (3 of 3 files). Show all

Unified · Split

3 files
  1. scripts/dev-seed.ts +6 −9
  2. scripts/lib/dev-seed.test.ts +36 −9
  3. scripts/lib/dev-seed.ts +24 −1
scripts/dev-seed.ts+6−9
@@ -17,13 +17,11 @@ import { Command, Flag } from "effect/unstable/cli"; · 16 unmodified lines
17 17
18import { resolveWorktreeT3Home } from "@t3tools/shared/devHome";18import { resolveWorktreeT3Home } from "@t3tools/shared/devHome";
19 19
20import { DevSeedError, seedDevDatabase } from "./lib/dev-seed.ts";20import { DevSeedError, MAX_SEED_THREAD_LIMIT, seedDevDatabase } from "./lib/dev-seed.ts";
21import { findRunningServerPid } from "./lib/server-runtime-probe.ts";21import { findRunningServerPid } from "./lib/server-runtime-probe.ts";
22 22
23const DEFAULT_THREAD_LIMIT = 25;23const DEFAULT_THREAD_LIMIT = 25;
24const DEFAULT_ACTIVITY_LIMIT = 200;24const DEFAULT_ACTIVITY_LIMIT = 200;
25/** SQLite's default SQLITE_MAX_VARIABLE_NUMBER; one bound parameter per thread. */
26const MAX_THREAD_LIMIT = 32_766;
27 25
28class DevSeedTargetError extends Schema.TaggedErrorClass<DevSeedTargetError>()(26class DevSeedTargetError extends Schema.TaggedErrorClass<DevSeedTargetError>()(
29 "DevSeedTargetError",27 "DevSeedTargetError",
@@ -140,16 +138,15 @@ const devSeedCli = Command.make("dev-seed", { · 110 unmodified lines
140 Flag.map(Option.getOrUndefined),138 Flag.map(Option.getOrUndefined),
141 ),139 ),
142 // Bounded below: SQLite reads a negative LIMIT as "no limit", which would140 // Bounded below: SQLite reads a negative LIMIT as "no limit", which would
143 // quietly turn a capped copy into a full-table one. Bounded above because141 // quietly turn a capped copy into a full-table one. The upper bound comes
144 // every selected thread becomes one bound parameter in the copy's `IN (...)`,142 // from the seeder itself, which binds one variable per selected thread —
145 // and SQLite's compiled-in ceiling is 32766 — past it the seed dies with143 // stating it separately here is what previously let the two drift apart.
146 // "too many SQL variables" after the target has already been emptied.
147 threads: Flag.integer("threads").pipe(144 threads: Flag.integer("threads").pipe(
148 Flag.withSchema(145 Flag.withSchema(
149 Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(MAX_THREAD_LIMIT)),146 Schema.Int.check(Schema.isGreaterThan(0), Schema.isLessThanOrEqualTo(MAX_SEED_THREAD_LIMIT)),
150 ),147 ),
151 Flag.withDescription(148 Flag.withDescription(
152 `How many recent threads to copy (default ${String(DEFAULT_THREAD_LIMIT)}, max ${String(MAX_THREAD_LIMIT)}).`,149 `How many recent threads to copy (default ${String(DEFAULT_THREAD_LIMIT)}, max ${String(MAX_SEED_THREAD_LIMIT)}).`,
153 ),150 ),
154 Flag.withDefault(DEFAULT_THREAD_LIMIT),151 Flag.withDefault(DEFAULT_THREAD_LIMIT),
155 ),152 ),
scripts/lib/dev-seed.test.ts+36−9
@@ -5,7 +5,7 @@ import * as NodeOS from "node:os"; · 4 unmodified lines
5import * as NodePath from "node:path";5import * as NodePath from "node:path";
6import * as NodeSqlite from "node:sqlite";6import * as NodeSqlite from "node:sqlite";
7 7
8import { DevSeedError, seedDevDatabase } from "./dev-seed.ts";8import { DevSeedError, MAX_SEED_THREAD_LIMIT, seedDevDatabase } from "./dev-seed.ts";
9 9
10const SEEDED_AT = "2026-07-26T00:00:00.000Z";10const SEEDED_AT = "2026-07-26T00:00:00.000Z";
11 11
@@ -511,23 +511,22 @@ describe("seedDevDatabase", () => { · 499 unmodified lines
511 });511 });
512 512
513 // Every statement keyed on thread ids must cost exactly one bound variable513 // Every statement keyed on thread ids must cost exactly one bound variable
514 // per thread: `--threads` is capped at SQLite's ceiling (32766), so a single514 // per thread: the limit *is* SQLite's ceiling, so a single extra binding
515 // extra binding makes the documented maximum fail — after the target has515 // makes the maximum fail — after the target has already been emptied. Runs
516 // already been emptied. Runs the real seed, since the point is that no code516 // the real seed, since the point is that no code path adds a binding, not
517 // path adds a binding, not that one hand-written statement doesn't.517 // that one hand-written statement doesn't.
518 it("seeds at the maximum thread limit without exhausting SQL variables", () => {518 it("seeds at the maximum thread limit without exhausting SQL variables", () => {
519 const MAX_THREADS = 32_766;
520 withDatabases(519 withDatabases(
521 ({ source, target }) => {520 ({ source, target }) => {
522 const summary = seedDevDatabase({521 const summary = seedDevDatabase({
523 sourceDbPath: source,522 sourceDbPath: source,
524 targetDbPath: target,523 targetDbPath: target,
525 threadLimit: MAX_THREADS,524 threadLimit: MAX_SEED_THREAD_LIMIT,
526 activityLimit: 1,525 activityLimit: 1,
527 seededAt: SEEDED_AT,526 seededAt: SEEDED_AT,
528 });527 });
529 528
530 assert.equal(summary.threads, MAX_THREADS);529 assert.equal(summary.threads, MAX_SEED_THREAD_LIMIT);
531 const [unsettled] = query<{ count: number }>(530 const [unsettled] = query<{ count: number }>(
532 target,531 target,
533 "SELECT COUNT(*) count FROM projection_turns WHERE state IN ('running','pending')",532 "SELECT COUNT(*) count FROM projection_turns WHERE state IN ('running','pending')",
@@ -536,10 +535,38 @@ describe("seedDevDatabase", () => { · 2 unmodified lines
536 },535 },
537 // Threads only: the per-thread tables are what drive the binding count,536 // Threads only: the per-thread tables are what drive the binding count,
538 // and skipping their rows keeps the fixture cheap to build.537 // and skipping their rows keeps the fixture cheap to build.
539 { threads: MAX_THREADS, activities: 0, minimalRows: true },538 { threads: MAX_SEED_THREAD_LIMIT, activities: 0, minimalRows: true },
540 );539 );
541 });540 });
542 541
542 // A caller that bypasses the CLI flag would otherwise discover the ceiling
543 // only after the target had been emptied and the copy rolled back.
544 it("refuses a thread limit above the ceiling before touching the target", () => {
545 withDatabases(({ source, target }) => {
546 assert.throws(
547 () =>
548 seedDevDatabase({
549 sourceDbPath: source,
550 targetDbPath: target,
551 threadLimit: MAX_SEED_THREAD_LIMIT + 1,
552 activityLimit: 10,
553 seededAt: SEEDED_AT,
554 }),
555 DevSeedError,
556 );
557
558 // The target's own row is still there: nothing was deleted.
559 const projects = query<{ project_id: string }>(
560 target,
561 "SELECT project_id FROM projection_projects",
562 );
563 assert.deepStrictEqual(
564 projects.map((row) => row.project_id),
565 ["stale"],
566 );
567 });
568 });
569
543 it("reports a source with nothing to copy", () => {570 it("reports a source with nothing to copy", () => {
544 withDatabases(571 withDatabases(
545 ({ source, target }) => {572 ({ source, target }) => {
scripts/lib/dev-seed.ts+24−1
@@ -21,10 +21,23 @@ import * as NodeSqlite from "node:sqlite"; · 20 unmodified lines
21 21
22import { PROJECTION_TABLES_IN_DEPENDENCY_ORDER, PROJECTOR_NAMES } from "./projection-tables.ts";22import { PROJECTION_TABLES_IN_DEPENDENCY_ORDER, PROJECTOR_NAMES } from "./projection-tables.ts";
23 23
24/**
25 * The most threads one call can copy: SQLite's default
26 * SQLITE_MAX_VARIABLE_NUMBER, because the statements below bind one variable
27 * per selected thread.
28 *
29 * Exported so the CLI's `--threads` bound comes from the code that actually has
30 * to satisfy it. When the two were stated separately, a statement that bound
31 * two extra literals put the CLI's own documented maximum over the ceiling —
32 * and it failed after the target had been emptied. Anything added to those
33 * statements has to stay free of per-call bindings, or this has to come down.
34 */
35export const MAX_SEED_THREAD_LIMIT = 32_766;
36
24export interface DevSeedOptions {37export interface DevSeedOptions {
25 readonly sourceDbPath: string;38 readonly sourceDbPath: string;
26 readonly targetDbPath: string;39 readonly targetDbPath: string;
27 /** How many recent threads to copy. */40 /** How many recent threads to copy. Capped at {@link MAX_SEED_THREAD_LIMIT}. */
28 readonly threadLimit: number;41 readonly threadLimit: number;
29 /**42 /**
30 * Newest activities kept per thread. The real table runs to six figures, and43 * Newest activities kept per thread. The real table runs to six figures, and
@@ -266,6 +279,16 @@ function settleUnfinishedTurns( · 235 unmodified lines
266}279}
267 280
268export function seedDevDatabase(options: DevSeedOptions): DevSeedSummary {281export function seedDevDatabase(options: DevSeedOptions): DevSeedSummary {
282 // Checked before anything is opened, and certainly before the target is
283 // emptied: overrunning the ceiling mid-copy destroys the target's data and
284 // rolls back without replacing it.
285 if (options.threadLimit > MAX_SEED_THREAD_LIMIT) {
286 throw new DevSeedError(
287 `cannot copy more than ${String(MAX_SEED_THREAD_LIMIT)} threads in one pass`,
288 "Each thread costs one SQLite bound variable, and that is the engine's limit.",
289 );
290 }
291
269 let source: NodeSqlite.DatabaseSync;292 let source: NodeSqlite.DatabaseSync;
270 try {293 try {
271 source = new NodeSqlite.DatabaseSync(options.sourceDbPath, { readOnly: true });294 source = new NodeSqlite.DatabaseSync(options.sourceDbPath, { readOnly: true });

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t