History

perf(storage): add bulk plugin-storage upserts (#1098)

* feat(fumadb): add bulk upsert queries * perf(sdk): upsert plugin storage bulk writes * test(openapi): update plugin storage mock facade Add collection-level bulk methods to the OpenAPI store test stub so it satisfies the expanded PluginStorageCollectionFacade interface. * fix(fumadb): validate bulk upsert conflict shapes * fix(fumadb): batch bounded D1 upserts atomically * fix(sdk): transact bulk plugin storage writes * Reconcile bulk plugin-storage upserts with the landed putMany API The collection-level putMany/removeMany the branch added duplicate no production consumer; both call sites use the facade. Drop them and keep the branch to its real contribution, the atomic upsert implementation. Add a case proving a mid-batch failure leaves pre-existing rows intact, and make the fault injector unconditional so removing the transaction fails the test on the data rather than silently disarming it. * Fix bulk upsert batching, predicate grouping, and memory conflict semantics * Make split bulk writes atomic and fail fast on parameter budget overflow * Quote excluded columns, serialize SQLite transaction scopes, and transact generic bulk upserts ---------

Saatvik Arya committed 2 months ago

Commit 02b52cd1 parent d7e4b7312 files · +1529 −86Browse files

Unified · Split

12 files
  1. .changeset/plugin-storage-bulk-upserts.md +7 −0
  2. packages/core/fumadb/src/adapters/drizzle/query.ts +293 −39
  3. packages/core/fumadb/src/adapters/drizzle/upsert-many-generic.test.ts +120 −0
  4. packages/core/fumadb/src/adapters/memory/index.ts +33 −0
  5. packages/core/fumadb/src/query/index.ts +16 −0
  6. packages/core/fumadb/src/query/orm/index.ts +183 −1
  7. packages/core/fumadb/src/query/orm/predicate-values-equal.test.ts +34 −0
  8. packages/core/fumadb/src/query/table-policy.test.ts +571 −5
  9. packages/core/sdk/src/executor.ts +52 −39
  10. packages/core/sdk/src/fuma-runtime.ts +1 −0
  11. packages/core/sdk/src/plugin-storage.test.ts +210 −2
  12. packages/core/sdk/src/test-config.ts +9 −0
.changeset/plugin-storage-bulk-upserts.md+7−0
@@ -0,0 +1,7 @@
1---
2"@executor-js/fumadb": patch
3"@executor-js/sdk": patch
4---
5
6Add a FumaDB bulk upsert query path and route plugin-storage bulk writes through
7it so existing rows are updated without delete/reinsert churn.
packages/core/fumadb/src/adapters/drizzle/query.ts+293−39
@@ -22,6 +22,77 @@ type P_DBType = PostgreSQL.PgDatabase< · 21 unmodified lines
22 22
23const CREATE_MANY_BATCH_SIZE = 500;23const CREATE_MANY_BATCH_SIZE = 500;
24 24
25// A multi-row write binds (rows * columns) parameters in one statement, and
26// engines cap bound parameters per statement (older SQLite: 999, Cloudflare
27// D1: 100). When the adapter does not advertise its limit, budget against the
28// conservative 999 floor so a wide table cannot overflow with "too many SQL
29// variables" on stricter engines.
30const DEFAULT_MAX_BOUND_PARAMETERS = 999;
31
32function parameterBoundedBatchSize(
33 table: AnyTable,
34 columnsPerRow: number,
35 reservedParameters: number,
36 maxBoundParameters: number | undefined
37): number {
38 const budget = maxBoundParameters ?? DEFAULT_MAX_BOUND_PARAMETERS;
39 const rowsPerStatement = Math.floor((budget - reservedParameters) / columnsPerRow);
40 if (rowsPerStatement < 1) {
41 // Even a single row cannot fit the advertised budget. Clamping to one row
42 // anyway would silently emit a statement the engine may reject, so fail
43 // fast and name the numbers instead.
44 const reservedNote =
45 reservedParameters > 0 ? ` and the predicate reserves ${reservedParameters} more` : "";
46 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: adapter rejects writes that cannot fit the engine's bound-parameter budget
47 throw new Error(
48 `[FumaDB Drizzle] Cannot write table "${table.ormName}": one row binds ${columnsPerRow} bound parameters${reservedNote}, which exceeds the ${budget}-parameter budget per statement.`
49 );
50 }
51 return Math.min(CREATE_MANY_BATCH_SIZE, rowsPerStatement);
52}
53
54// SQLite transaction scopes run as raw BEGIN/SAVEPOINT statements on one
55// shared connection, so two scopes started concurrently would interleave
56// their control statements: two top-level BEGINs collide, and sibling
57// savepoints at one nesting depth reuse each other's names — one sibling's
58// ROLLBACK TO can undo work the other already released. Serialize scope
59// execution per connection handle and nesting depth: same-depth scopes run
60// one after another, while a parent scope at depth N can still open its
61// child scope at depth N + 1 without deadlocking. SQLite is single-writer,
62// so this serialization costs no real concurrency.
63const transactionScopeQueues = new WeakMap<object, Map<number, Promise<void>>>();
64
65async function runSerializedScope<T>(
66 handle: object,
67 depth: number,
68 fn: () => Promise<T>,
69): Promise<T> {
70 let queues = transactionScopeQueues.get(handle);
71 if (!queues) {
72 queues = new Map();
73 transactionScopeQueues.set(handle, queues);
74 }
75 const previous = queues.get(depth) ?? Promise.resolve();
76 let release!: () => void;
77 queues.set(
78 depth,
79 new Promise<void>((resolve) => {
80 release = resolve;
81 }),
82 );
83 await previous;
84 // oxlint-disable-next-line executor/no-try-catch-or-throw -- scope release must survive the wrapped failure
85 try {
86 return await fn();
87 } finally {
88 release();
89 }
90}
91
92// Savepoint names are invocation-unique as defense in depth: even if two
93// scopes ever interleave, a ROLLBACK TO can only target its own savepoint.
94let savepointSequence = 0;
95
25function buildWhere(96function buildWhere(
26 toDrizzle: (col: AnyColumn) => ColumnType,97 toDrizzle: (col: AnyColumn) => ColumnType,
27 condition: Condition98 condition: Condition
@@ -121,6 +192,16 @@ function buildWhere( · 93 unmodified lines
121 );192 );
122}193}
123 194
195function countConditionParameters(condition: Condition): number {
196 if (condition.type === ConditionType.Compare) {
197 if (condition.b instanceof Column) return 0;
198 if (Array.isArray(condition.b)) return condition.b.length;
199 return 1;
200 }
201 if (condition.type === ConditionType.Not) return countConditionParameters(condition.item);
202 return condition.items.reduce((count, item) => count + countConditionParameters(item), 0);
203}
204
124function mapValues(205function mapValues(
125 values: Record<string, unknown>,206 values: Record<string, unknown>,
126 table: AnyTable207 table: AnyTable
@@ -171,7 +252,8 @@ export function fromDrizzle( · 44 unmodified lines
171 _db: unknown,252 _db: unknown,
172 provider: SQLProvider,253 provider: SQLProvider,
173 interactiveTransactions: boolean = true,254 interactiveTransactions: boolean = true,
174 maxBoundParameters?: number255 maxBoundParameters?: number,
256 transactionDepth: number = 0
175): AbstractQuery<AnySchema> {257): AbstractQuery<AnySchema> {
176 const [db, drizzleTables] = parseDrizzle(_db);258 const [db, drizzleTables] = parseDrizzle(_db);
177 259
@@ -195,6 +277,41 @@ export function fromDrizzle( · 17 unmodified lines
195 throw new Error("[FumaDB Drizzle] Database cannot execute raw transaction statements.");277 throw new Error("[FumaDB Drizzle] Database cannot execute raw transaction statements.");
196 }278 }
197 279
280 // Runs `fn` atomically when the engine allows it. SQLite drivers get raw
281 // BEGIN/COMMIT on the shared connection — or a SAVEPOINT when this adapter
282 // instance already lives inside a transaction, so nested use (e.g. a bulk
283 // upsert splitting into predicate groups inside a caller's transaction)
284 // does not issue a second BEGIN. Other providers get a driver transaction
285 // whose handle must be used for every statement inside `fn`. When the
286 // engine rejects interactive transactions (Cloudflare D1), statements
287 // auto-commit — that engine constraint is documented on `transaction`.
288 async function runAtomically<T>(fn: (handle: typeof db) => Promise<T>): Promise<T> {
289 if (!interactiveTransactions) {
290 return fn(db);
291 }
292 if (provider === "sqlite") {
293 return runSerializedScope(db, transactionDepth, async () => {
294 savepointSequence += 1;
295 const savepoint = transactionDepth > 0 ? `fumadb_tx_${savepointSequence}` : undefined;
296 await executeRaw(savepoint ? `SAVEPOINT ${savepoint}` : "BEGIN");
297 try {
298 const result = await fn(db);
299 await executeRaw(savepoint ? `RELEASE SAVEPOINT ${savepoint}` : "COMMIT");
300 return result;
301 } catch (e) {
302 if (savepoint) {
303 await executeRaw(`ROLLBACK TO SAVEPOINT ${savepoint}`);
304 await executeRaw(`RELEASE SAVEPOINT ${savepoint}`);
305 } else {
306 await executeRaw("ROLLBACK");
307 }
308 throw e;
309 }
310 });
311 }
312 return db.transaction((tx) => fn(tx as unknown as typeof db));
313 }
314
198 function toDrizzle(v: AnyTable): TableType {315 function toDrizzle(v: AnyTable): TableType {
199 const out = drizzleTables[v.names.drizzle];316 const out = drizzleTables[v.names.drizzle];
200 if (out) return out;317 if (out) return out;
@@ -303,6 +420,117 @@ export function fromDrizzle( · 102 unmodified lines
303 await this.createMany(table, [v.create]);420 await this.createMany(table, [v.create]);
304 }421 }
305 },422 },
423 async upsertMany(table, v) {
424 if (v.values.length === 0) return;
425 if (v.target.length === 0) {
426 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: adapter rejects invalid upsert shape
427 throw new Error("[FumaDB] upsertMany requires at least one target column.");
428 }
429 if (v.update.length === 0) {
430 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: adapter rejects invalid upsert shape
431 throw new Error("[FumaDB] upsertMany requires at least one update column.");
432 }
433 if (provider !== "sqlite" && provider !== "postgresql") {
434 // This path issues several statements per row, so a failure halfway
435 // through must not leave a prefix of the rows committed. These
436 // providers have native driver transactions; `transaction` routes
437 // every per-row statement through one of them.
438 await this.transaction(async (scoped) => {
439 for (const value of v.values) {
440 const targetCondition: Condition = {
441 type: ConditionType.And,
442 items: v.target.map((column) => ({
443 type: ConditionType.Compare,
444 a: column,
445 operator: "=",
446 b: value[column.ormName],
447 })),
448 };
449 await scoped.internal.upsert(table, {
450 where: v.where
451 ? { type: ConditionType.And, items: [targetCondition, v.where] }
452 : targetCondition,
453 update: Object.fromEntries(
454 v.update.map((column) => [column.ormName, value[column.ormName]]),
455 ),
456 create: value,
457 });
458 }
459 });
460 return;
461 }
462
463 const drizzleTable = toDrizzle(table);
464 const values = v.values.map((value) => mapValues(value, table));
465 const where = v.where ? buildWhere(toDrizzleColumn, v.where) : undefined;
466 const whereParameters = v.where ? countConditionParameters(v.where) : 0;
467 const columnsPerRow = values.length > 0 ? Math.max(1, Object.keys(values[0]!).length) : 1;
468 const batchSize = parameterBoundedBatchSize(
469 table,
470 columnsPerRow,
471 whereParameters,
472 maxBoundParameters,
473 );
474 const target = v.target.map((column) => drizzleTable[column.names.drizzle]);
475 const set = Object.fromEntries(
476 v.update.map((column) => [
477 column.names.drizzle,
478 // `sql.identifier` quotes the physical column name; a raw
479 // interpolation would emit e.g. `excluded.display-name` unquoted,
480 // which the engine parses as an expression.
481 Drizzle.sql`excluded.${Drizzle.sql.identifier(column.names.sql)}`,
482 ]),
483 );
484
485 const buildStatements = (handle: typeof db): unknown[] => {
486 const statements: unknown[] = [];
487 for (let i = 0; i < values.length; i += batchSize) {
488 const batch = values.slice(i, i + batchSize);
489 const insert = handle.insert(drizzleTable).values(batch) as unknown as {
490 onConflictDoUpdate: (input: {
491 readonly target: typeof target;
492 readonly set: typeof set;
493 readonly where?: typeof where;
494 }) => unknown;
495 };
496 statements.push(
497 insert.onConflictDoUpdate({
498 target,
499 set,
500 ...(where === undefined ? {} : { where }),
501 }),
502 );
503 }
504 return statements;
505 };
506 const executeStatements = async (handle: typeof db) => {
507 for (const statement of buildStatements(handle)) {
508 await statement;
509 }
510 };
511 const statementCount = Math.ceil(values.length / batchSize);
512
513 // D1 rejects interactive transactions but its native batch API executes
514 // prepared statements as one transaction. Drizzle exposes that API on
515 // the database handle, so keep parameter-bounded upserts atomic instead
516 // of auto-committing each statement independently.
517 const nativeBatch = db as unknown as {
518 readonly batch?: (statements: readonly unknown[]) => Promise<unknown>;
519 };
520 if (!interactiveTransactions && statementCount > 1 && nativeBatch.batch) {
521 await nativeBatch.batch(buildStatements(db));
522 return;
523 }
524
525 if (statementCount === 1) {
526 await executeStatements(db);
527 return;
528 }
529
530 // One logical upsert split into several parameter-bounded statements
531 // stays atomic: run them inside one transaction.
532 await runAtomically(executeStatements);
533 },
306 async findMany(table, v) {534 async findMany(table, v) {
307 return (535 return (
308 await db.query[table.names.drizzle].findMany(buildQueryConfig(table, v))536 await db.query[table.names.drizzle].findMany(buildQueryConfig(table, v))
@@ -353,43 +581,48 @@ export function fromDrizzle( · 44 unmodified lines
353 },581 },
354 582
355 async createMany(table, values) {583 async createMany(table, values) {
584 if (values.length === 0) return [];
356 const idField = table.getIdColumn().names.drizzle;585 const idField = table.getIdColumn().names.drizzle;
357 const drizzleTable = toDrizzle(table);586 const drizzleTable = toDrizzle(table);
358 values = values.map((v) => mapValues(v, table));587 values = values.map((v) => mapValues(v, table));
359 // A multi-row insert binds (rows * columns) parameters in one statement.588 const columnsPerRow = Math.max(1, Object.keys(values[0]!).length);
360 // Some engines cap bound parameters per query (Cloudflare D1: 100), so589 const batchSize = parameterBoundedBatchSize(table, columnsPerRow, 0, maxBoundParameters);
361 // size the batch by PARAMETER count, not row count — otherwise a wide
362 // table (e.g. tools) overflows with "too many SQL variables". Engines
363 // without a tight cap keep the row-count batch.
364 const columnsPerRow = values.length > 0 ? Math.max(1, Object.keys(values[0]!).length) : 1;
365 const batchSize = maxBoundParameters
366 ? Math.max(1, Math.min(CREATE_MANY_BATCH_SIZE, Math.floor(maxBoundParameters / columnsPerRow)))
367 : CREATE_MANY_BATCH_SIZE;
368 const batches: (typeof values)[] = [];590 const batches: (typeof values)[] = [];
369 for (let i = 0; i < values.length; i += batchSize) {591 for (let i = 0; i < values.length; i += batchSize) {
370 batches.push(values.slice(i, i + batchSize));592 batches.push(values.slice(i, i + batchSize));
371 }593 }
372 594
373 if (provider === "sqlite" || provider === "postgresql") {595 const insertBatches = async (handle: typeof db): Promise<{ _id: unknown }[]> => {
374 const out: { _id: unknown }[] = [];596 if (provider === "sqlite" || provider === "postgresql") {
375 for (const batch of batches) {597 const out: { _id: unknown }[] = [];
376 out.push(598 for (const batch of batches) {
377 ...(await (db as unknown as P_DBType)599 out.push(
378 .insert(drizzleTable as unknown as P_TableType)600 ...(await (handle as unknown as P_DBType)
379 .values(batch)601 .insert(drizzleTable as unknown as P_TableType)
380 .returning({602 .values(batch)
381 _id: (drizzleTable as unknown as P_TableType)[idField],603 .returning({
382 })),604 _id: (drizzleTable as unknown as P_TableType)[idField],
383 );605 })),
606 );
607 }
608 return out;
384 }609 }
385 return out;
386 }
387 610
388 const results: Record<string, unknown>[] = [];611 const results: Record<string, unknown>[] = [];
389 for (const batch of batches) {612 for (const batch of batches) {
390 results.push(...(await db.insert(drizzleTable).values(batch).$returningId()));613 results.push(...(await handle.insert(drizzleTable).values(batch).$returningId()));
391 }614 }
392 return results.map((result) => ({ _id: result[idField] }));615 return results.map((result) => ({ _id: result[idField] }));
616 };
617
618 if (batches.length === 1) return insertBatches(db);
619 // One logical insert split into parameter-bounded statements must stay
620 // atomic: a later batch's constraint failure rolls back the earlier
621 // batches. (Engines without interactive transactions auto-commit each
622 // statement; `createMany` needs the inserted ids back, which the native
623 // batch API's result shape does not guarantee across drivers, so those
624 // engines keep sequential statements — their documented constraint.)
625 return runAtomically(insertBatches);
393 },626 },
394 627
395 async deleteMany(table, v) {628 async deleteMany(table, v) {
@@ -409,23 +642,44 @@ export function fromDrizzle( · 13 unmodified lines
409 // each statement auto-commits, so there is no atomic rollback (the642 // each statement auto-commits, so there is no atomic rollback (the
410 // engine's constraint, not ours). libSQL/Postgres keep real transactions.643 // engine's constraint, not ours). libSQL/Postgres keep real transactions.
411 if (!interactiveTransactions) {644 if (!interactiveTransactions) {
412 return run(fromDrizzle(schema, _db, provider, interactiveTransactions, maxBoundParameters));645 return run(
646 fromDrizzle(
647 schema,
648 _db,
649 provider,
650 interactiveTransactions,
651 maxBoundParameters,
652 transactionDepth
653 )
654 );
413 }655 }
414 656
415 if (provider === "sqlite") {657 if (provider === "sqlite") {
416 await executeRaw("BEGIN");658 return runAtomically(() =>
417 try {659 run(
418 const result = await run(fromDrizzle(schema, _db, provider, interactiveTransactions, maxBoundParameters));660 fromDrizzle(
419 await executeRaw("COMMIT");661 schema,
420 return result;662 _db,
421 } catch (e) {663 provider,
422 await executeRaw("ROLLBACK");664 interactiveTransactions,
423 throw e;665 maxBoundParameters,
424 }666 transactionDepth + 1
667 )
668 )
669 );
425 }670 }
426 671
427 return db.transaction((tx) =>672 return db.transaction((tx) =>
428 run(fromDrizzle(schema, tx, provider, interactiveTransactions, maxBoundParameters))673 run(
674 fromDrizzle(
675 schema,
676 tx,
677 provider,
678 interactiveTransactions,
679 maxBoundParameters,
680 transactionDepth + 1
681 )
682 )
429 );683 );
430 },684 },
431 });685 });
packages/core/fumadb/src/adapters/drizzle/upsert-many-generic.test.ts+120−0
@@ -0,0 +1,120 @@
1import { expect, test } from "@effect/vitest";
2
3import { column, idColumn, schema, table } from "../../schema";
4import { fromDrizzle } from "./query";
5
6// The generic bulk-upsert path (providers without ON CONFLICT support, e.g.
7// MySQL) issues several statements per row. The unit harness has no MySQL
8// server, so a recording fake of the Drizzle handle asserts the transaction
9// routing instead: every per-row statement must run on the driver
10// transaction's handle, and a failing row must roll the earlier rows back.
11const v1 = schema({
12 version: "1.0.0",
13 tables: {
14 rows: table("generic_rows", {
15 id: idColumn("id", "varchar(255)"),
16 value: column("value", "string"),
17 }),
18 },
19});
20
21interface FakeDbOptions {
22 // A row whose `value` matches makes its INSERT fail.
23 readonly failOnValue?: string;
24}
25
26const createFakeMysqlDb = (options: FakeDbOptions = {}) => {
27 const committed: Record<string, unknown>[] = [];
28 const events: string[] = [];
29 const fakeTable = { id: { name: "id" }, value: { name: "value" } };
30
31 const makeHandle = (label: string, sink: Record<string, unknown>[]) => ({
32 _: { fullSchema: { rows: fakeTable } },
33 select: () => {
34 const builder = {
35 from: () => builder,
36 limit: () => builder,
37 where: () => builder,
38 execute: async (): Promise<Record<string, unknown>[]> => [],
39 };
40 return builder;
41 },
42 insert: () => ({
43 values: (rows: Record<string, unknown>[]) => ({
44 $returningId: async () => {
45 events.push(`${label}:insert`);
46 for (const row of rows) {
47 if (options.failOnValue !== undefined && row["value"] === options.failOnValue) {
48 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- test forces a row's statement to fail
49 throw new Error("statement failure");
50 }
51 sink.push(row);
52 }
53 return rows.map((row) => ({ id: row["id"] }));
54 },
55 }),
56 }),
57 transaction: async <T>(callback: (tx: unknown) => Promise<T>): Promise<T> => {
58 events.push("transaction:begin");
59 const staged: Record<string, unknown>[] = [];
60 const tx = makeHandle("tx", staged);
61 // oxlint-disable-next-line executor/no-try-catch-or-throw -- fake driver mirrors commit/rollback
62 try {
63 const result = await callback(tx);
64 committed.push(...staged);
65 events.push("transaction:commit");
66 return result;
67 } catch (error) {
68 events.push("transaction:rollback");
69 throw error;
70 }
71 },
72 });
73
74 // Statements on the root handle auto-commit, exactly like a driver outside
75 // an explicit transaction.
76 return { db: makeHandle("root", committed), committed, events };
77};
78
79const upsertAll = async (db: unknown, values: readonly { id: string; value: string }[]) => {
80 const orm = fromDrizzle(v1, db, "mysql");
81 const rows = v1.tables.rows;
82 expect(orm.internal.upsertMany).toBeDefined();
83 await orm.internal.upsertMany?.(rows, {
84 target: [rows.columns.id],
85 update: [rows.columns.value],
86 values: values.map((value) => ({ ...value })),
87 });
88};
89
90test("generic bulk upsert runs every row inside one driver transaction", async () => {
91 const { db, committed, events } = createFakeMysqlDb();
92
93 await upsertAll(db, [
94 { id: "r1", value: "a" },
95 { id: "r2", value: "b" },
96 { id: "r3", value: "c" },
97 ]);
98
99 expect(committed.map((row) => row["id"])).toEqual(["r1", "r2", "r3"]);
100 expect(events).toContain("transaction:begin");
101 expect(events).toContain("transaction:commit");
102 // No row statement may run on the auto-committing root handle.
103 expect(events).not.toContain("root:insert");
104});
105
106test("generic bulk upsert rolls earlier rows back when a later row fails", async () => {
107 const { db, committed, events } = createFakeMysqlDb({ failOnValue: "poison" });
108
109 await expect(
110 upsertAll(db, [
111 { id: "r1", value: "a" },
112 { id: "r2", value: "b" },
113 { id: "r3", value: "poison" },
114 ]),
115 ).rejects.toThrow("statement failure");
116
117 // A single logical bulk upsert must not leave a partial prefix committed.
118 expect(committed).toEqual([]);
119 expect(events).toContain("transaction:rollback");
120});
packages/core/fumadb/src/adapters/memory/index.ts+33−0
@@ -174,6 +174,39 @@ export function memoryAdapter(options: MemoryAdapterOptions = {}): FumaDBAdapter · 173 unmodified lines
174 }174 }
175 await this.create(table, v.create);175 await this.create(table, v.create);
176 },176 },
177 async upsertMany(table, v) {
178 if (v.target.length === 0) {
179 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: adapter rejects invalid upsert shape
180 throw new Error("[FumaDB] upsertMany requires at least one target column.");
181 }
182 if (v.update.length === 0) {
183 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: adapter rejects invalid upsert shape
184 throw new Error("[FumaDB] upsertMany requires at least one update column.");
185 }
186 for (const value of v.values) {
187 // Mirror SQL `ON CONFLICT ... DO UPDATE ... WHERE`: the unique
188 // target alone detects the conflict, and the predicate only
189 // decides whether the conflicting row may be updated. A
190 // policy-excluded conflict skips the row; it never inserts a
191 // duplicate of the unique target.
192 const conflicting = tableRows(db, table).find((row) =>
193 v.target.every((column) => row[column.ormName] === value[column.ormName]),
194 );
195 if (conflicting) {
196 if (!matchesCondition(conflicting, v.where)) continue;
197 Object.assign(
198 conflicting,
199 cloneValue(
200 Object.fromEntries(
201 v.update.map((column) => [column.ormName, value[column.ormName]]),
202 ),
203 ),
204 );
205 continue;
206 }
207 await this.create(table, value);
208 }
209 },
177 async create(table, values) {210 async create(table, values) {
178 const row = applyDefaults(table, values);211 const row = applyDefaults(table, values);
179 tableRows(db, table).push(row);212 tableRows(db, table).push(row);
packages/core/fumadb/src/query/index.ts+16−0
@@ -174,6 +174,22 @@ export interface AbstractQuery<S extends AnySchema> { · 173 unmodified lines
174 }174 }
175 ) => Promise<void>;175 ) => Promise<void>;
176 176
177 /**
178 * Bulk upsert rows by a unique column target.
179 *
180 * Adapters with native conflict support should implement this as one or more
181 * `INSERT ... ON CONFLICT ... DO UPDATE` statements. Other adapters may fall
182 * back to per-row `upsert`.
183 */
184 upsertMany: <TableName extends keyof S["tables"]>(
185 table: TableName,
186 v: {
187 target: (keyof S["tables"][TableName]["columns"])[];
188 update: (keyof S["tables"][TableName]["columns"])[];
189 values: TableToInsertValues<S["tables"][TableName]>[];
190 }
191 ) => Promise<void>;
192
177 /**193 /**
178 * Note: you cannot update the id of a row, some databases don't support that (including MongoDB).194 * Note: you cannot update the id of a row, some databases don't support that (including MongoDB).
179 */195 */
packages/core/fumadb/src/query/orm/index.ts+183−1
@@ -4,6 +4,7 @@ import type { · 3 unmodified lines
4 AnySchema,4 AnySchema,
5 AnyTable,5 AnyTable,
6} from "../../schema";6} from "../../schema";
7import { Column } from "../../schema";
7import type {8import type {
8 AbstractQuery,9 AbstractQuery,
9 AnySelectClause,10 AnySelectClause,
@@ -12,7 +13,12 @@ import type { · 2 unmodified lines
12 JoinBuilder,13 JoinBuilder,
13 OrderBy,14 OrderBy,
14} from "..";15} from "..";
15import { buildCondition, createBuilder, type Condition } from "../condition-builder";16import {
17 buildCondition,
18 createBuilder,
19 type Condition,
20 ConditionType,
21} from "../condition-builder";
16 22
17export interface CompiledJoin {23export interface CompiledJoin {
18 relation: AnyRelation;24 relation: AnyRelation;
@@ -231,6 +237,68 @@ const applyUpdatePolicies = async ( · 212 unmodified lines
231 return nextWhere;237 return nextWhere;
232};238};
233 239
240// Structural equality over predicate values. Policy predicates may carry any
241// column value shape — string, number, bigint, boolean, null, Date, binary,
242// JSON objects, and arrays of those — so serializing them to string keys
243// (e.g. JSON.stringify) either throws (bigint) or collides (values with the
244// same serialized form). Comparing structurally is unambiguous; a false
245// negative only splits a group and never merges rows under the wrong
246// predicate.
247export const predicateValuesEqual = (a: unknown, b: unknown): boolean => {
248 if (Object.is(a, b)) return true;
249 if (a instanceof Date || b instanceof Date) {
250 return a instanceof Date && b instanceof Date && a.getTime() === b.getTime();
251 }
252 if (a instanceof Uint8Array || b instanceof Uint8Array) {
253 return (
254 a instanceof Uint8Array &&
255 b instanceof Uint8Array &&
256 a.length === b.length &&
257 a.every((byte, index) => byte === b[index])
258 );
259 }
260 if (Array.isArray(a) || Array.isArray(b)) {
261 if (!Array.isArray(a) || !Array.isArray(b) || a.length !== b.length) return false;
262 // Compare by index instead of `.every`, which skips holes in sparse
263 // arrays — `Array(1)` would otherwise equal `[123]` and merge rows under
264 // the wrong predicate group. A hole and a present element are unequal.
265 for (let index = 0; index < a.length; index += 1) {
266 const aHas = index in a;
267 if (aHas !== index in b) return false;
268 if (aHas && !predicateValuesEqual(a[index], b[index])) return false;
269 }
270 return true;
271 }
272 if (isRecord(a) && isRecord(b)) {
273 const aKeys = Object.keys(a);
274 const bKeys = Object.keys(b);
275 return (
276 aKeys.length === bKeys.length &&
277 aKeys.every((key) => Object.hasOwn(b, key) && predicateValuesEqual(a[key], b[key]))
278 );
279 }
280 return false;
281};
282
283const conditionsEqual = (a: Condition | undefined, b: Condition | undefined): boolean => {
284 if (a === undefined || b === undefined) return a === b;
285 if (a.type === ConditionType.Compare || b.type === ConditionType.Compare) {
286 if (a.type !== ConditionType.Compare || b.type !== ConditionType.Compare) return false;
287 if (a.a !== b.a || a.operator !== b.operator) return false;
288 if (a.b instanceof Column || b.b instanceof Column) return a.b === b.b;
289 return predicateValuesEqual(a.b, b.b);
290 }
291 if (a.type === ConditionType.Not || b.type === ConditionType.Not) {
292 if (a.type !== ConditionType.Not || b.type !== ConditionType.Not) return false;
293 return conditionsEqual(a.item, b.item);
294 }
295 return (
296 a.type === b.type &&
297 a.items.length === b.items.length &&
298 a.items.every((item, index) => conditionsEqual(item, b.items[index]))
299 );
300};
301
234const applyDeletePolicies = async (302const applyDeletePolicies = async (
235 table: AnyTable,303 table: AnyTable,
236 where: Condition | undefined,304 where: Condition | undefined,
@@ -284,6 +352,16 @@ export interface ORMAdapter<S extends AnySchema = AnySchema> { · 47 unmodified lines
284 },352 },
285 ) => Promise<void>;353 ) => Promise<void>;
286 354
355 upsertMany?: (
356 table: AnyTable,
357 v: {
358 target: AnyColumn[];
359 update: AnyColumn[];
360 values: Record<string, unknown>[];
361 where?: Condition;
362 },
363 ) => Promise<void>;
364
287 create: (365 create: (
288 table: AnyTable,366 table: AnyTable,
289 values: Record<string, unknown>,367 values: Record<string, unknown>,
@@ -367,6 +445,110 @@ export function toORM<S extends AnySchema>( · 77 unmodified lines
367 ...options,445 ...options,
368 });446 });
369 },447 },
448 async upsertMany(name, { target, update, values }) {
449 const table = toTable(name);
450 if (values.length === 0) return;
451 if (target.length === 0) {
452 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: public query rejects invalid upsert shape
453 throw new Error("[FumaDB] upsertMany requires at least one target column.");
454 }
455 if (update.length === 0) {
456 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: public query rejects invalid upsert shape
457 throw new Error("[FumaDB] upsertMany requires at least one update column.");
458 }
459
460 const targetColumns = target.map((columnName) => {
461 const column = table.columns[columnName as string];
462 if (!column) throw new Error(`[FumaDB] unknown column name ${String(columnName)}.`);
463 return column;
464 });
465 const updateColumns = update.map((columnName) => {
466 const column = table.columns[columnName as string];
467 if (!column) throw new Error(`[FumaDB] unknown column name ${String(columnName)}.`);
468 return column;
469 });
470
471 const builder = createBuilder(table.columns);
472 const permittedRows: {
473 readonly value: Record<string, unknown>;
474 readonly where: Condition | undefined;
475 }[] = [];
476 for (const value of values) {
477 const updateValues = Object.fromEntries(
478 updateColumns.map((column) => [column.ormName, value[column.ormName]]),
479 );
480 const constrainedWhere = await applyUpdatePolicies(
481 table,
482 undefined,
483 updateValues,
484 context,
485 "upsert",
486 value,
487 );
488 if (constrainedWhere === false) continue;
489 await runCreatePolicies(table, value, context);
490 permittedRows.push({ value, where: constrainedWhere });
491 }
492 if (permittedRows.length === 0) return;
493
494 if (internal.upsertMany) {
495 const groups: {
496 readonly where: Condition | undefined;
497 readonly values: Record<string, unknown>[];
498 }[] = [];
499 for (const row of permittedRows) {
500 const group = groups.find((candidate) => conditionsEqual(candidate.where, row.where));
501 if (group) {
502 group.values.push(row.value);
503 } else {
504 groups.push({ where: row.where, values: [row.value] });
505 }
506 }
507 const runGroups = async (adapter: ORMAdapter<S>): Promise<void> => {
508 if (!adapter.upsertMany) {
509 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: a transaction adapter must mirror the base adapter's upsertMany support
510 throw new Error("[FumaDB] Transaction adapter does not support upsertMany.");
511 }
512 for (const group of groups) {
513 await adapter.upsertMany(table, {
514 target: targetColumns,
515 update: updateColumns,
516 values: group.values,
517 where: group.where,
518 });
519 }
520 };
521
522 // A single group is one adapter call, which is as atomic as the
523 // engine allows. Multiple predicate groups are separate adapter
524 // calls, so run them inside one transaction: a later group's failure
525 // must not leave earlier groups committed.
526 if (groups.length === 1) {
527 await runGroups(internal);
528 return;
529 }
530 await internal.transaction(async (transactionInstance) => {
531 await runGroups(transactionInstance.internal);
532 });
533 return;
534 }
535
536 for (const row of permittedRows) {
537 const value = row.value;
538 const targetWhere = builder.and(
539 ...targetColumns.map((column) => builder(column.ormName, "=", value[column.ormName])),
540 );
541 const where = builder.and(targetWhere, row.where ?? true);
542 if (where === false) continue;
543 await internal.upsert(table, {
544 where: where === true ? undefined : where,
545 update: Object.fromEntries(
546 updateColumns.map((column) => [column.ormName, value[column.ormName]]),
547 ),
548 create: value,
549 });
550 }
551 },
370 async create(name, values) {552 async create(name, values) {
371 const table = toTable(name);553 const table = toTable(name);
372 await runCreatePolicies(table, values, context);554 await runCreatePolicies(table, values, context);
packages/core/fumadb/src/query/orm/predicate-values-equal.test.ts+34−0
@@ -0,0 +1,34 @@
1import { describe, expect, it } from "@effect/vitest";
2
3import { predicateValuesEqual } from "./index";
4
5describe("predicateValuesEqual", () => {
6 it("treats a sparse-array hole as unequal to a present element, in both orders", () => {
7 // oxlint-disable-next-line unicorn/no-new-array -- the sparse array is the case under test
8 expect(predicateValuesEqual(Array(1), [123])).toBe(false);
9 // oxlint-disable-next-line unicorn/no-new-array -- the sparse array is the case under test
10 expect(predicateValuesEqual([123], Array(1))).toBe(false);
11 });
12
13 it("treats arrays with matching holes as equal", () => {
14 // oxlint-disable-next-line unicorn/no-new-array -- the sparse array is the case under test
15 expect(predicateValuesEqual(Array(2), Array(2))).toBe(true);
16 });
17
18 it("keeps structural equality for dense arrays", () => {
19 expect(
20 predicateValuesEqual([BigInt(7), new Date(5), Number.NaN], [BigInt(7), new Date(5), Number.NaN]),
21 ).toBe(true);
22 expect(predicateValuesEqual([1, 2], [1, 3])).toBe(false);
23 expect(predicateValuesEqual([1, undefined], [1, undefined])).toBe(true);
24 });
25
26 it("keeps the existing Date, Uint8Array, and record handling", () => {
27 expect(predicateValuesEqual(new Date(5), new Date(5))).toBe(true);
28 expect(predicateValuesEqual(new Date(5), new Date(6))).toBe(false);
29 expect(predicateValuesEqual(new Uint8Array([1, 2]), new Uint8Array([1, 2]))).toBe(true);
30 expect(predicateValuesEqual(new Uint8Array([1, 2]), new Uint8Array([1, 3]))).toBe(false);
31 expect(predicateValuesEqual({ a: BigInt(1) }, { a: BigInt(1) })).toBe(true);
32 expect(predicateValuesEqual({ a: 1 }, { a: 1, b: 2 })).toBe(false);
33 });
34});
packages/core/fumadb/src/query/table-policy.test.ts+571−5
@@ -8,6 +8,7 @@ import { · 7 unmodified lines
8 createDrizzleRuntimeSchemaSqlFromTables,8 createDrizzleRuntimeSchemaSqlFromTables,
9 drizzleAdapter,9 drizzleAdapter,
10} from "@executor-js/fumadb/adapters/drizzle";10} from "@executor-js/fumadb/adapters/drizzle";
11import { memoryAdapter } from "@executor-js/fumadb/adapters/memory";
11import { withQueryContext, type AbstractQuery } from "@executor-js/fumadb/query";12import { withQueryContext, type AbstractQuery } from "@executor-js/fumadb/query";
12import { column, idColumn, schema, table } from "@executor-js/fumadb/schema";13import { column, idColumn, schema, table } from "@executor-js/fumadb/schema";
13 14
@@ -16,6 +17,7 @@ interface TenantPolicyContext { · 2 unmodified lines
16 readonly deniedTables: ReadonlySet<string>;17 readonly deniedTables: ReadonlySet<string>;
17 readonly marker: string;18 readonly marker: string;
18 readonly observed: string[];19 readonly observed: string[];
20 readonly allowedRegionId?: bigint;
19}21}
20 22
21const observe = (context: TenantPolicyContext, event: string) => {23const observe = (context: TenantPolicyContext, event: string) => {
@@ -103,16 +105,68 @@ const comments = table("policy_comments", { · 81 unmodified lines
103 },105 },
104});106});
105 107
108// Policy predicate carries a bigint value; string-keyed grouping of these
109// predicates (e.g. JSON.stringify) throws on bigint.
110const quotas = table("policy_quotas", {
111 id: idColumn("id", "varchar(255)"),
112 region: column("region", "bigint"),
113 label: column("label", "string"),
114}).policy<TenantPolicyContext>({
115 name: "tenant.quotas",
116 onUpdate: ({ builder, context }) => builder("region", "=", context.allowedRegionId ?? BigInt(-1)),
117});
118
119// The update policy predicate depends on the row itself, so rows with
120// different `shard` values compile to different predicates and split one
121// upsertMany call into separate predicate groups.
122const shards = table("policy_shards", {
123 id: idColumn("id", "varchar(255)"),
124 shard: column("shard", "string"),
125 authorId: column("author_id", "varchar(255)"),
126}).policy<TenantPolicyContext>({
127 name: "tenant.shards",
128 onUpdate: ({ builder, set }) => builder("shard", "=", set.shard ?? ""),
129});
130
131// Ten columns, so a row-count-sized batch would bind ten parameters per row.
132const wideRows = table("policy_wide_rows", {
133 id: idColumn("id", "varchar(255)"),
134 c1: column("c1", "string"),
135 c2: column("c2", "string"),
136 c3: column("c3", "string"),
137 c4: column("c4", "string"),
138 c5: column("c5", "string"),
139 c6: column("c6", "string"),
140 c7: column("c7", "string"),
141 c8: column("c8", "string"),
142 c9: column("c9", "string"),
143});
144
145// The physical column name needs identifier quoting; the bulk upsert conflict
146// SET clause references it as `excluded.<name>` and must quote it.
147const quotedNames = table("policy_quoted_names", {
148 id: idColumn("id", "varchar(255)"),
149 displayName: column("display-name", "string"),
150});
151
106const v1 = schema({152const v1 = schema({
107 version: "1.0.0",153 version: "1.0.0",
108 tables: {154 tables: {
109 authors,155 authors,
110 posts,156 posts,
111 comments,157 comments,
158 quotas,
159 shards,
160 wideRows,
161 quotedNames,
112 },162 },
113 relations: {163 relations: {
114 authors: ({ many }) => ({164 authors: ({ many }) => ({
115 posts: many("posts"),165 posts: many("posts"),
166 shards: many("shards"),
167 }),
168 shards: ({ one }) => ({
169 author: one("authors", ["authorId", "id"]).foreignKey(),
116 }),170 }),
117 posts: ({ one, many }) => ({171 posts: ({ one, many }) => ({
118 author: one("authors", ["authorId", "id"]).foreignKey(),172 author: one("authors", ["authorId", "id"]).foreignKey(),
@@ -142,7 +196,10 @@ const makeContext = ( · 23 unmodified lines
142 observed: [],196 observed: [],
143});197});
144 198
145const makeHarness = async () => {199const makeHarness = async (options?: {
200 readonly nativeBatch?: boolean;
201 readonly maxBoundParameters?: number;
202}) => {
146 const sqlite = new Database(":memory:");203 const sqlite = new Database(":memory:");
147 sqlite.pragma("foreign_keys = ON");204 sqlite.pragma("foreign_keys = ON");
148 const runtimeSchema = createDrizzleRuntimeSchemaFromTables({205 const runtimeSchema = createDrizzleRuntimeSchemaFromTables({
@@ -151,7 +208,25 @@ const makeHarness = async () => { · 2 unmodified lines
151 version: "1.0.0",208 version: "1.0.0",
152 provider: "sqlite",209 provider: "sqlite",
153 });210 });
154 const drizzleDb = drizzle(sqlite, { schema: runtimeSchema });211 const statements: { readonly sql: string; readonly paramCount: number }[] = [];
212 const drizzleDb = drizzle(sqlite, {
213 schema: runtimeSchema,
214 logger: {
215 logQuery: (query: string, params: unknown[]) => {
216 statements.push({ sql: query, paramCount: params.length });
217 },
218 },
219 });
220 let batchCalls = 0;
221
222 if (options?.nativeBatch) {
223 Object.assign(drizzleDb, {
224 batch: async (queries: readonly { run: () => unknown }[]) => {
225 batchCalls += 1;
226 return sqlite.transaction(() => queries.map((query) => query.run()))();
227 },
228 });
229 }
155 230
156 for (const statement of createDrizzleRuntimeSchemaSqlFromTables({231 for (const statement of createDrizzleRuntimeSchemaSqlFromTables({
157 tables: v1.tables,232 tables: v1.tables,
@@ -166,11 +241,16 @@ const makeHarness = async () => { · 8 unmodified lines
166 drizzleAdapter({241 drizzleAdapter({
167 db: drizzleDb,242 db: drizzleDb,
168 provider: "sqlite",243 provider: "sqlite",
244 interactiveTransactions: options?.nativeBatch ? false : undefined,
245 maxBoundParameters: options?.maxBoundParameters,
169 }),246 }),
170 );247 );
171 248
172 return {249 return {
173 orm: client.orm("1.0.0"),250 orm: client.orm("1.0.0"),
251 getBatchCalls: () => batchCalls,
252 getStatements: (): readonly { readonly sql: string; readonly paramCount: number }[] =>
253 statements,
174 close: async () => {254 close: async () => {
175 sqlite.close();255 sqlite.close();
176 },256 },
@@ -179,11 +259,45 @@ const makeHarness = async () => { · 2 unmodified lines
179 259
180const useHarness = <A>(run: (orm: TablePolicyQuery) => Promise<A>) =>260const useHarness = <A>(run: (orm: TablePolicyQuery) => Promise<A>) =>
181 Effect.acquireUseRelease(261 Effect.acquireUseRelease(
182 Effect.promise(makeHarness),262 Effect.promise(() => makeHarness()),
263 ({ orm }) => Effect.promise(() => run(orm)),
264 ({ close }) => Effect.promise(close),
265 );
266
267const useStatementHarness = <A>(
268 run: (harness: {
269 readonly orm: TablePolicyQuery;
270 readonly getStatements: () => readonly { readonly sql: string; readonly paramCount: number }[];
271 }) => Promise<A>,
272) =>
273 Effect.acquireUseRelease(
274 Effect.promise(() => makeHarness()),
275 (harness) => Effect.promise(() => run(harness)),
276 ({ close }) => Effect.promise(close),
277 );
278
279const useBudgetHarness = <A>(
280 maxBoundParameters: number,
281 run: (orm: TablePolicyQuery) => Promise<A>,
282) =>
283 Effect.acquireUseRelease(
284 Effect.promise(() => makeHarness({ maxBoundParameters })),
183 ({ orm }) => Effect.promise(() => run(orm)),285 ({ orm }) => Effect.promise(() => run(orm)),
184 ({ close }) => Effect.promise(close),286 ({ close }) => Effect.promise(close),
185 );287 );
186 288
289const useNativeBatchHarness = <A>(
290 run: (harness: {
291 readonly orm: TablePolicyQuery;
292 readonly getBatchCalls: () => number;
293 }) => Promise<A>,
294) =>
295 Effect.acquireUseRelease(
296 Effect.promise(() => makeHarness({ nativeBatch: true, maxBoundParameters: 8 })),
297 (harness) => Effect.promise(() => run(harness)),
298 ({ close }) => Effect.promise(close),
299 );
300
187const seedTenants = async (orm: TablePolicyQuery) => {301const seedTenants = async (orm: TablePolicyQuery) => {
188 const seed = withQueryContext(orm, makeContext(["tenant-a", "tenant-b"], "seed"));302 const seed = withQueryContext(orm, makeContext(["tenant-a", "tenant-b"], "seed"));
189 303
@@ -382,6 +496,24 @@ describe("FumaDB table policies", () => { · 192 unmodified lines
382 title: "A Three",496 title: "A Three",
383 },497 },
384 });498 });
499 await tenantA.upsertMany("posts", {
500 target: ["id"],
501 update: ["title"],
502 values: [
503 {
504 id: "post-a-1",
505 tenantId: "tenant-a",
506 authorId: "author-a",
507 title: "tenant-a-bulk-upserted",
508 },
509 {
510 id: "post-a-4",
511 tenantId: "tenant-a",
512 authorId: "author-a",
513 title: "A Four",
514 },
515 ],
516 });
385 517
386 await expect(518 await expect(
387 tenantA.findMany("posts", {519 tenantA.findMany("posts", {
@@ -391,7 +523,7 @@ describe("FumaDB table policies", () => { · 3 unmodified lines
391 ).resolves.toEqual([523 ).resolves.toEqual([
392 {524 {
393 id: "post-a-1",525 id: "post-a-1",
394 title: "tenant-a-updated",526 title: "tenant-a-bulk-upserted",
395 },527 },
396 {528 {
397 id: "post-a-2",529 id: "post-a-2",
@@ -401,6 +533,10 @@ describe("FumaDB table policies", () => { · 3 unmodified lines
401 id: "post-a-3",533 id: "post-a-3",
402 title: "A Three",534 title: "A Three",
403 },535 },
536 {
537 id: "post-a-4",
538 title: "A Four",
539 },
404 ]);540 ]);
405 541
406 expect(tenantAContext.observed).toEqual(542 expect(tenantAContext.observed).toEqual(
@@ -460,6 +596,395 @@ describe("FumaDB table policies", () => { · 53 unmodified lines
460 }),596 }),
461 );597 );
462 598
599 it.effect("rejects invalid bulk upsert conflict shapes", () =>
600 useHarness(async (orm) => {
601 await seedTenants(orm);
602 const tenantA = withQueryContext(orm, makeContext(["tenant-a"], "tenant-a"));
603 const values = [
604 {
605 id: "post-a-bulk-upsert",
606 tenantId: "tenant-a",
607 authorId: "author-a",
608 title: "A bulk upsert",
609 },
610 ];
611
612 await expect(
613 tenantA.upsertMany("posts", {
614 target: [],
615 update: ["title"],
616 values,
617 }),
618 ).rejects.toThrow("[FumaDB] upsertMany requires at least one target column.");
619
620 await expect(
621 tenantA.upsertMany("posts", {
622 target: ["id"],
623 update: [],
624 values,
625 }),
626 ).rejects.toThrow("[FumaDB] upsertMany requires at least one update column.");
627 }),
628 );
629
630 it.effect("keeps every bulk upsert statement inside the bound-variable budget", () =>
631 useStatementHarness(async ({ orm, getStatements }) => {
632 // 250 rows * 10 columns = 2,500 bound variables in a single statement —
633 // over older SQLite's 999-variable cap. The adapter advertises no limit
634 // here, so batching must fall back to the conservative 999 budget.
635 const values = Array.from({ length: 250 }, (_, index) => ({
636 id: `wide-${String(index).padStart(3, "0")}`,
637 c1: `value-${index}-1`,
638 c2: `value-${index}-2`,
639 c3: `value-${index}-3`,
640 c4: `value-${index}-4`,
641 c5: `value-${index}-5`,
642 c6: `value-${index}-6`,
643 c7: `value-${index}-7`,
644 c8: `value-${index}-8`,
645 c9: `value-${index}-9`,
646 }));
647
648 await orm.upsertMany("wideRows", {
649 target: ["id"],
650 update: ["c1"],
651 values,
652 });
653
654 const inserts = getStatements().filter((statement) =>
655 statement.sql.toLowerCase().startsWith("insert into"),
656 );
657 expect(inserts.length).toBeGreaterThanOrEqual(3);
658 for (const statement of inserts) {
659 expect(statement.paramCount).toBeLessThanOrEqual(999);
660 }
661 await expect(orm.count("wideRows")).resolves.toBe(250);
662 }),
663 );
664
665 it.effect("bulk upserts a column whose physical name needs identifier quoting", () =>
666 useHarness(async (orm) => {
667 await orm.createMany("quotedNames", [{ id: "row-1", displayName: "before" }]);
668
669 // The conflict SET clause references the update column through the
670 // `excluded` pseudo-table; an unquoted physical name like
671 // `display-name` is a syntax error there.
672 await orm.upsertMany("quotedNames", {
673 target: ["id"],
674 update: ["displayName"],
675 values: [
676 { id: "row-1", displayName: "after" },
677 { id: "row-2", displayName: "created" },
678 ],
679 });
680
681 await expect(
682 orm.findMany("quotedNames", {
683 select: ["id", "displayName"],
684 orderBy: ["id", "asc"],
685 }),
686 ).resolves.toEqual([
687 { id: "row-1", displayName: "after" },
688 { id: "row-2", displayName: "created" },
689 ]);
690 }),
691 );
692
693 it.effect("keeps a successful sibling nested transaction when the other rolls back", () =>
694 useHarness(async (orm) => {
695 const writer = withQueryContext(orm, makeContext(["tenant-a"], "siblings"));
696
697 await writer.transaction(async (tx) => {
698 // Two sibling savepoint scopes started concurrently on the one shared
699 // SQLite connection: the failing sibling's ROLLBACK TO must not undo
700 // work the successful sibling already released.
701 const failing = tx.transaction(async (inner) => {
702 await inner.create("authors", {
703 id: "author-rolled-back",
704 tenantId: "tenant-a",
705 name: "Rolled Back",
706 });
707 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- test forces one sibling to roll back
708 throw new Error("sibling failure");
709 });
710 const succeeding = tx.transaction(async (inner) => {
711 await inner.create("authors", {
712 id: "author-committed",
713 tenantId: "tenant-a",
714 name: "Committed",
715 });
716 });
717
718 const [failed, succeeded] = await Promise.allSettled([failing, succeeding]);
719 expect(failed.status).toBe("rejected");
720 expect(succeeded.status).toBe("fulfilled");
721 });
722
723 await expect(
724 writer.findMany("authors", {
725 select: ["id"],
726 where: (builder) =>
727 builder.or(
728 builder("id", "=", "author-rolled-back"),
729 builder("id", "=", "author-committed"),
730 ),
731 }),
732 ).resolves.toEqual([{ id: "author-committed" }]);
733 }),
734 );
735
736 it.effect("bulk upserts through a policy predicate that contains a bigint", () =>
737 useHarness(async (orm) => {
738 const region = withQueryContext(orm, {
739 ...makeContext(["tenant-a"], "region"),
740 allowedRegionId: BigInt(7),
741 });
742
743 await region.createMany("quotas", [
744 { id: "quota-in-region", region: BigInt(7), label: "before" },
745 { id: "quota-out-of-region", region: BigInt(9), label: "before" },
746 ]);
747
748 await region.upsertMany("quotas", {
749 target: ["id"],
750 update: ["label"],
751 values: [
752 { id: "quota-in-region", region: BigInt(7), label: "after" },
753 { id: "quota-out-of-region", region: BigInt(9), label: "after" },
754 { id: "quota-created", region: BigInt(7), label: "created" },
755 ],
756 });
757
758 await expect(
759 region.findMany("quotas", {
760 select: ["id", "label"],
761 orderBy: ["id", "asc"],
762 }),
763 ).resolves.toEqual([
764 { id: "quota-created", label: "created" },
765 { id: "quota-in-region", label: "after" },
766 { id: "quota-out-of-region", label: "before" },
767 ]);
768 }),
769 );
770
771 it.effect("skips policy-excluded conflicts identically on drizzle and memory adapters", () => {
772 // `post-b-1` already exists for tenant-b, so its unique target conflicts,
773 // and the tenant-a update policy excludes the conflicting row. SQL detects
774 // the conflict first and the predicate only gates the update, so the row
775 // is skipped — never duplicated.
776 const upsertAcrossPolicyExcludedConflict = async (orm: TablePolicyQuery) => {
777 await seedTenants(orm);
778 const tenantA = withQueryContext(orm, makeContext(["tenant-a"], "tenant-a"));
779 await tenantA.upsertMany("posts", {
780 target: ["id"],
781 update: ["title"],
782 values: [
783 {
784 id: "post-b-1",
785 tenantId: "tenant-a",
786 authorId: "author-a",
787 title: "hijack attempt",
788 },
789 {
790 id: "post-a-9",
791 tenantId: "tenant-a",
792 authorId: "author-a",
793 title: "A Nine",
794 },
795 ],
796 });
797 const allTenants = withQueryContext(orm, makeContext(["tenant-a", "tenant-b"], "all"));
798 return allTenants.findMany("posts", {
799 select: ["id", "tenantId", "title"],
800 orderBy: ["id", "asc"],
801 });
802 };
803
804 const expected = [
805 { id: "post-a-1", tenantId: "tenant-a", title: "A One" },
806 { id: "post-a-2", tenantId: "tenant-a", title: "A Two" },
807 { id: "post-a-9", tenantId: "tenant-a", title: "A Nine" },
808 { id: "post-b-1", tenantId: "tenant-b", title: "B One" },
809 ];
810
811 return useHarness(async (orm) => {
812 await expect(upsertAcrossPolicyExcludedConflict(orm)).resolves.toEqual(expected);
813
814 const memoryOrm = tablePolicyDB.client(memoryAdapter()).orm("1.0.0");
815 await expect(upsertAcrossPolicyExcludedConflict(memoryOrm)).resolves.toEqual(expected);
816 });
817 });
818
819 it.effect("rolls back every bounded upsert statement when a native batch fails", () =>
820 useNativeBatchHarness(async ({ orm, getBatchCalls }) => {
821 await seedTenants(orm);
822 const tenantA = withQueryContext(orm, makeContext(["tenant-a"], "tenant-a"));
823
824 await expect(
825 tenantA.upsertMany("posts", {
826 target: ["id"],
827 update: ["title"],
828 values: [
829 {
830 id: "post-a-batch-1",
831 tenantId: "tenant-a",
832 authorId: "author-a",
833 title: "A batch one",
834 },
835 {
836 id: "post-a-batch-2",
837 tenantId: "tenant-a",
838 authorId: "author-a",
839 title: "A batch two",
840 },
841 {
842 id: "post-a-batch-invalid",
843 tenantId: "tenant-a",
844 authorId: "missing-author",
845 title: "Must roll back",
846 },
847 ],
848 }),
849 ).rejects.toThrow();
850
851 expect(getBatchCalls()).toBe(1);
852 await expect(
853 tenantA.findMany("posts", {
854 where: (builder) => builder("id", "starts with", "post-a-batch-"),
855 select: ["id"],
856 }),
857 ).resolves.toEqual([]);
858 }),
859 );
860
861 it.effect("rolls back earlier predicate groups when a later group fails", () =>
862 useHarness(async (orm) => {
863 await seedTenants(orm);
864 const writer = withQueryContext(orm, makeContext(["tenant-a"], "groups"));
865
866 // Two distinct per-row predicates split this call into two internal
867 // groups. The second group's row references a missing author, so its
868 // insert violates the foreign key — the first group's rows must roll
869 // back with it instead of staying committed.
870 await expect(
871 writer.upsertMany("shards", {
872 target: ["id"],
873 update: ["shard"],
874 values: [
875 { id: "shard-a-1", shard: "a", authorId: "author-a" },
876 { id: "shard-b-1", shard: "b", authorId: "missing-author" },
877 ],
878 }),
879 ).rejects.toThrow();
880
881 await expect(orm.count("shards")).resolves.toBe(0);
882 }),
883 );
884
885 it.effect("rolls back earlier createMany batches when a later batch fails", () =>
886 useBudgetHarness(8, async (orm) => {
887 await seedTenants(orm);
888 const tenantA = withQueryContext(orm, makeContext(["tenant-a"], "tenant-a"));
889
890 // Four columns per row against an 8-parameter budget forces two-row
891 // batches, so this insert runs as two statements. The last row reuses
892 // the seeded `post-a-1` id, so the second statement violates the
893 // primary key — the first statement's rows must roll back with it.
894 await expect(
895 tenantA.createMany("posts", [
896 { id: "post-a-n1", tenantId: "tenant-a", authorId: "author-a", title: "N1" },
897 { id: "post-a-n2", tenantId: "tenant-a", authorId: "author-a", title: "N2" },
898 { id: "post-a-n3", tenantId: "tenant-a", authorId: "author-a", title: "N3" },
899 { id: "post-a-1", tenantId: "tenant-a", authorId: "author-a", title: "Duplicate" },
900 ]),
901 ).rejects.toThrow();
902
903 await expect(
904 tenantA.findMany("posts", {
905 where: (builder) => builder("id", "starts with", "post-a-n"),
906 select: ["id"],
907 }),
908 ).resolves.toEqual([]);
909 }),
910 );
911
912 it.effect("fails fast when a single row exceeds the bound-parameter budget", () =>
913 useBudgetHarness(8, async (orm) => {
914 await expect(
915 orm.createMany("wideRows", [
916 {
917 id: "wide-overflow",
918 c1: "1",
919 c2: "2",
920 c3: "3",
921 c4: "4",
922 c5: "5",
923 c6: "6",
924 c7: "7",
925 c8: "8",
926 c9: "9",
927 },
928 ]),
929 ).rejects.toThrow(
930 'one row binds 10 bound parameters, which exceeds the 8-parameter budget',
931 );
932
933 await expect(orm.count("wideRows")).resolves.toBe(0);
934 }),
935 );
936
937 it.effect("fails fast when reserved predicate parameters consume the budget", () =>
938 useBudgetHarness(4, async (orm) => {
939 const seed = withQueryContext(orm, makeContext(["tenant-a"], "seed"));
940 await seed.createMany("authors", [{ id: "author-a", tenantId: "tenant-a", name: "Ada" }]);
941
942 const tenantA = withQueryContext(orm, makeContext(["tenant-a"], "tenant-a"));
943 // A posts row alone fits the 4-parameter budget exactly, but the update
944 // policy predicate reserves one more bound parameter per statement.
945 await expect(
946 tenantA.upsertMany("posts", {
947 target: ["id"],
948 update: ["title"],
949 values: [{ id: "post-a-r1", tenantId: "tenant-a", authorId: "author-a", title: "R1" }],
950 }),
951 ).rejects.toThrow("the predicate reserves 1 more");
952
953 await expect(tenantA.count("posts")).resolves.toBe(0);
954 }),
955 );
956
957 it.effect("applies intra-call conflicts identically on drizzle and memory adapters", () => {
958 // The second value's unique target conflicts with a row created by the
959 // first value in the SAME upsertMany call: both adapters must update the
960 // freshly created row instead of duplicating or dropping it.
961 const upsertIntraCallConflict = async (orm: TablePolicyQuery) => {
962 await seedTenants(orm);
963 const tenantA = withQueryContext(orm, makeContext(["tenant-a"], "tenant-a"));
964 await tenantA.upsertMany("posts", {
965 target: ["id"],
966 update: ["title"],
967 values: [
968 { id: "post-a-dup", tenantId: "tenant-a", authorId: "author-a", title: "first write" },
969 { id: "post-a-dup", tenantId: "tenant-a", authorId: "author-a", title: "second write" },
970 ],
971 });
972 return tenantA.findMany("posts", {
973 where: (builder) => builder("id", "=", "post-a-dup"),
974 select: ["id", "title"],
975 });
976 };
977
978 const expected = [{ id: "post-a-dup", title: "second write" }];
979
980 return useHarness(async (orm) => {
981 await expect(upsertIntraCallConflict(orm)).resolves.toEqual(expected);
982
983 const memoryOrm = tablePolicyDB.client(memoryAdapter()).orm("1.0.0");
984 await expect(upsertIntraCallConflict(memoryOrm)).resolves.toEqual(expected);
985 });
986 });
987
463 it.effect("fails closed when a query wrapper does not forward context rebinding", () =>988 it.effect("fails closed when a query wrapper does not forward context rebinding", () =>
464 useHarness(async (orm) => {989 useHarness(async (orm) => {
465 const wrapped = { ...orm };990 const wrapped = { ...orm };
@@ -471,7 +996,7 @@ describe("FumaDB table policies", () => { · 5 unmodified lines
471 );996 );
472 997
473 it.effect(998 it.effect(
474 "rejects out-of-context writes across createMany, updateMany, upsert, and transactions",999 "rejects out-of-context writes across createMany, updateMany, upsert, upsertMany, and transactions",
475 () =>1000 () =>
476 useHarness(async (orm) => {1001 useHarness(async (orm) => {
477 await seedTenants(orm);1002 await seedTenants(orm);
@@ -524,6 +1049,47 @@ describe("FumaDB table policies", () => { · 46 unmodified lines
524 }),1049 }),
525 ).rejects.toThrow("tenant tenant-b is not allowed for posts");1050 ).rejects.toThrow("tenant tenant-b is not allowed for posts");
526 1051
1052 await expect(
1053 tenantA.upsertMany("posts", {
1054 target: ["id"],
1055 update: ["title"],
1056 values: [
1057 {
1058 id: "post-a-bulk-upsert",
1059 tenantId: "tenant-a",
1060 authorId: "author-a",
1061 title: "A bulk upsert",
1062 },
1063 {
1064 id: "post-b-bulk-upsert",
1065 tenantId: "tenant-b",
1066 authorId: "author-b",
1067 title: "B bulk upsert",
1068 },
1069 ],
1070 }),
1071 ).rejects.toThrow("tenant tenant-b is not allowed for posts");
1072 await expect(
1073 tenantA.findFirst("posts", {
1074 where: (builder) => builder("id", "=", "post-a-bulk-upsert"),
1075 }),
1076 ).resolves.toBeNull();
1077
1078 await expect(
1079 tenantA.upsertMany("posts", {
1080 target: ["id"],
1081 update: ["tenantId"],
1082 values: [
1083 {
1084 id: "post-a-1",
1085 tenantId: "tenant-b",
1086 authorId: "author-b",
1087 title: "tenant move",
1088 },
1089 ],
1090 }),
1091 ).rejects.toThrow("tenant tenant-b is not allowed for posts");
1092
527 await expect(1093 await expect(
528 tenantA.transaction(async (tx) => {1094 tenantA.transaction(async (tx) => {
529 await tx.create("posts", {1095 await tx.create("posts", {
packages/core/sdk/src/executor.ts+52−39
@@ -196,7 +196,6 @@ import { makeShapeMemory, observedShapeToJsonSchema, SHAPE_MEMORY_PLUGIN_ID } fr · 195 unmodified lines
196import { isUnauthorizedToolFailure } from "./auth-tool-failure";196import { isUnauthorizedToolFailure } from "./auth-tool-failure";
197 197
198const PLUGIN_STORAGE_DELETE_KEY_BATCH_SIZE = 90;198const PLUGIN_STORAGE_DELETE_KEY_BATCH_SIZE = 90;
199const PLUGIN_STORAGE_CREATE_ROW_BATCH_SIZE = 90;
200const MAX_APPROVAL_ARGUMENT_PREVIEW_CHARS = 4_000;199const MAX_APPROVAL_ARGUMENT_PREVIEW_CHARS = 4_000;
201 200
202// ---------------------------------------------------------------------------201// ---------------------------------------------------------------------------
@@ -1075,6 +1074,14 @@ type LooseStorageDb = { · 872 unmodified lines
1075 tableName: string,1074 tableName: string,
1076 rows: readonly Record<string, unknown>[],1075 rows: readonly Record<string, unknown>[],
1077 ) => Promise<readonly unknown[]>;1076 ) => Promise<readonly unknown[]>;
1077 readonly upsertMany: (
1078 tableName: string,
1079 options: {
1080 readonly target: readonly string[];
1081 readonly update: readonly string[];
1082 readonly values: readonly Record<string, unknown>[];
1083 },
1084 ) => Promise<void>;
1078 readonly deleteMany: (tableName: string, options?: unknown) => Promise<void>;1085 readonly deleteMany: (tableName: string, options?: unknown) => Promise<void>;
1079 readonly findFirst: (1086 readonly findFirst: (
1080 tableName: string,1087 tableName: string,
@@ -1090,6 +1097,8 @@ type LooseStorageDb = { · 9 unmodified lines
1090const asLooseStorageDb = (db: unknown): LooseStorageDb => db as LooseStorageDb;1097const asLooseStorageDb = (db: unknown): LooseStorageDb => db as LooseStorageDb;
1091 1098
1092const makeCoreDb = (fuma: ReturnType<typeof makeFumaClient>) => ({1099const makeCoreDb = (fuma: ReturnType<typeof makeFumaClient>) => ({
1100 transaction: <A, E>(effect: Effect.Effect<A, E>): Effect.Effect<A, E | StorageFailure> =>
1101 fuma.transaction(effect),
1093 count: <TName extends CoreTableName>(1102 count: <TName extends CoreTableName>(
1094 tableName: TName,1103 tableName: TName,
1095 options?: { readonly where?: CoreWhere },1104 options?: { readonly where?: CoreWhere },
@@ -1111,6 +1120,19 @@ const makeCoreDb = (fuma: ReturnType<typeof makeFumaClient>) => ({ · 15 unmodified lines
1111 : fuma1120 : fuma
1112 .use(`${tableName}.createMany`, (db) => asLooseStorageDb(db).createMany(tableName, rows))1121 .use(`${tableName}.createMany`, (db) => asLooseStorageDb(db).createMany(tableName, rows))
1113 .pipe(Effect.asVoid),1122 .pipe(Effect.asVoid),
1123 upsertMany: <TName extends CoreTableName>(
1124 tableName: TName,
1125 options: {
1126 readonly target: readonly string[];
1127 readonly update: readonly string[];
1128 readonly values: readonly Record<string, unknown>[];
1129 },
1130 ): Effect.Effect<void, StorageFailure> =>
1131 options.values.length === 0
1132 ? Effect.void
1133 : fuma.use(`${tableName}.upsertMany`, (db) =>
1134 asLooseStorageDb(db).upsertMany(tableName, options),
1135 ),
1114 deleteMany: <TName extends CoreTableName>(1136 deleteMany: <TName extends CoreTableName>(
1115 tableName: TName,1137 tableName: TName,
1116 options: { readonly where?: CoreWhere } = {},1138 options: { readonly where?: CoreWhere } = {},
@@ -1439,42 +1461,33 @@ const makePluginStorageFacade = (input: { · 322 unmodified lines
1439 readonly data: unknown;1461 readonly data: unknown;
1440 }[],1462 }[],
1441 ) =>1463 ) =>
1442 Effect.gen(function* () {1464 input.core.transaction(
1443 const os = ownerSubject(owner);1465 Effect.gen(function* () {
1444 if (!os) {1466 const os = ownerSubject(owner);
1445 return yield* new StorageError({1467 if (!os) {
1446 message: `Cannot write plugin storage for owner "user": executor has no subject.`,1468 return yield* new StorageError({
1447 cause: undefined,1469 message: `Cannot write plugin storage for owner "user": executor has no subject.`,
1448 });1470 cause: undefined,
1449 }1471 });
1450 const entriesById = new Map(1472 }
1451 entries.map((entry) => [1473 const entriesById = new Map(
1452 pluginStorageId({1474 entries.map((entry) => [
1453 pluginId: input.pluginId,1475 pluginStorageId({
1454 collection: entry.collection,1476 pluginId: input.pluginId,
1455 key: entry.key,1477 collection: entry.collection,
1456 }),1478 key: entry.key,
1457 entry,1479 }),
1458 ]),1480 entry,
1459 );1481 ]),
1460 const uniqueEntries = [...entriesById.values()];
1461 if (uniqueEntries.length === 0) return;
1462
1463 yield* deleteManyImpl(owner, os.subject, uniqueEntries);
1464
1465 const now = new Date();
1466 for (
1467 let offset = 0;
1468 offset < uniqueEntries.length;
1469 offset += PLUGIN_STORAGE_CREATE_ROW_BATCH_SIZE
1470 ) {
1471 const batchEntries = uniqueEntries.slice(
1472 offset,
1473 offset + PLUGIN_STORAGE_CREATE_ROW_BATCH_SIZE,
1474 );1482 );
1475 yield* input.core.createMany(1483 const uniqueEntries = [...entriesById.values()];
1476 "plugin_storage",1484 if (uniqueEntries.length === 0) return;
1477 batchEntries.map((entry) => ({1485
1486 const now = new Date();
1487 yield* input.core.upsertMany("plugin_storage", {
1488 target: ["tenant", "owner", "subject", "plugin_id", "collection", "key"],
1489 update: ["data", "updated_at"],
1490 values: uniqueEntries.map((entry) => ({
1478 tenant,1491 tenant,
1479 owner: os.owner,1492 owner: os.owner,
1480 subject: os.subject,1493 subject: os.subject,
@@ -1485,9 +1498,9 @@ const makePluginStorageFacade = (input: { · 4 unmodified lines
1485 created_at: now,1498 created_at: now,
1486 updated_at: now,1499 updated_at: now,
1487 })),1500 })),
1488 );1501 });
1489 }1502 }),
1490 });1503 );
1491 1504
1492 const removeManyImpl = (1505 const removeManyImpl = (
1493 owner: Owner,1506 owner: Owner,
packages/core/sdk/src/fuma-runtime.ts+1−0
@@ -285,6 +285,7 @@ const makeSafeFumaQuery = <TSchema extends AnySchema>( · 284 unmodified lines
285 db.transaction((transactionDb) => run(makeSafeFumaQuery(transactionDb, options))),285 db.transaction((transactionDb) => run(makeSafeFumaQuery(transactionDb, options))),
286 updateMany: (name, value) => db.updateMany(table(name), value),286 updateMany: (name, value) => db.updateMany(table(name), value),
287 upsert: (name, value) => db.upsert(table(name), value),287 upsert: (name, value) => db.upsert(table(name), value),
288 upsertMany: (name, value) => db.upsertMany(table(name), value),
288 };289 };
289 290
290 return Object.freeze(query);291 return Object.freeze(query);
packages/core/sdk/src/plugin-storage.test.ts+210−2
@@ -1,7 +1,8 @@
1import { describe, expect, it } from "@effect/vitest";1import { describe, expect, it } from "@effect/vitest";
2import { Cause, Effect, Exit, Schema } from "effect";2import { Cause, Effect, Exit, Schema } from "effect";
3 3
4import { StorageError } from "./fuma-runtime";4import { createExecutor } from "./executor";
5import { StorageError, type FumaDb } from "./fuma-runtime";
5import { Owner } from "./ids";6import { Owner } from "./ids";
6import { definePlugin } from "./plugin";7import { definePlugin } from "./plugin";
7import {8import {
@@ -10,7 +11,7 @@ import { · 2 unmodified lines
10 type PluginStorageCollectionQueryInput,11 type PluginStorageCollectionQueryInput,
11 type PluginStorageCollectionWhere,12 type PluginStorageCollectionWhere,
12} from "./plugin-storage";13} from "./plugin-storage";
13import { makeTestExecutor } from "./testing";14import { makeTestConfig, makeTestExecutor } from "./testing";
14 15
15const ToolCall = Schema.Struct({16const ToolCall = Schema.Struct({
16 runId: Schema.String,17 runId: Schema.String,
@@ -113,6 +114,55 @@ const call = (input: { · 96 unmodified lines
113 durationMs: input.durationMs ?? 0,114 durationMs: input.durationMs ?? 0,
114});115});
115 116
117// A FumaDB that commits the FIRST row of a multi-row `plugin_storage` bulk
118// write and then fails. Injecting the fault mid-write, rather than before it,
119// is what makes the two rollback cases below meaningful: the row is really on
120// disk when the failure lands, so only the enclosing transaction can take it
121// back. The fault is unconditional (not armed by entering a transaction) so
122// that dropping the transaction is a visible failure and not a silently
123// disarmed test.
124const failPluginStorageBulkWriteAfterFirstRow = (db: FumaDb): FumaDb => {
125 const wrap = (source: FumaDb): FumaDb =>
126 new Proxy(source, {
127 get(target, property, receiver) {
128 if (property === "withContext") {
129 const withContext = target.withContext;
130 return withContext === undefined
131 ? undefined
132 : (context: unknown) => wrap(withContext(context));
133 }
134 if (property === "transaction") {
135 const transaction: FumaDb["transaction"] = (run) =>
136 target.transaction((transactionDb) => run(wrap(transactionDb)));
137 return transaction;
138 }
139 if (property === "upsertMany") {
140 const upsertMany: FumaDb["upsertMany"] = async (table, options) => {
141 if (table !== "plugin_storage" || options.values.length < 2) {
142 return target.upsertMany(table, options);
143 }
144
145 await target.upsertMany(table, {
146 ...options,
147 values: options.values.slice(0, 1),
148 });
149 // oxlint-disable-next-line executor/no-promise-reject -- boundary: fault-injecting FumaDB adapter must reject to exercise transaction rollback
150 return Promise.reject(
151 new StorageError({
152 message: "Injected plugin storage bulk-write failure.",
153 cause: undefined,
154 }),
155 );
156 };
157 return upsertMany;
158 }
159 return Reflect.get(target, property, receiver);
160 },
161 });
162
163 return wrap(db);
164};
165
116describe("plugin storage collections", () => {166describe("plugin storage collections", () => {
117 it.effect("queries declared indexes through the executor's SQLite FumaDB target", () =>167 it.effect("queries declared indexes through the executor's SQLite FumaDB target", () =>
118 Effect.gen(function* () {168 Effect.gen(function* () {
@@ -222,6 +272,164 @@ describe("plugin storage collections", () => { · 103 unmodified lines
222 }),272 }),
223 );273 );
224 274
275 it.effect("stores and overwrites every row when a bulk write spans multiple batches", () =>
276 Effect.gen(function* () {
277 const executor = yield* makeTestExecutor({
278 backend: "sqlite",
279 plugins: [executionHistoryPlugin] as const,
280 });
281 // A plugin_storage row binds ~9 values, so 300 rows exceed one
282 // 999-bound-variable statement budget and must span several batches.
283 const entries = (status: "ok" | "failed") =>
284 Array.from({ length: 300 }, (_, index) => ({
285 key: `batched-call-${String(index).padStart(3, "0")}`,
286 data: call({
287 runId: "run-batched",
288 toolId: "browser",
289 status,
290 startedAt: new Date(Date.UTC(2026, 4, 29, 12, 0, index)).toISOString(),
291 }),
292 }));
293
294 yield* executor.executionHistory.recordMany("org", entries("ok"));
295 const total = yield* executor.executionHistory.count({
296 where: { runId: "run-batched" },
297 });
298 expect(total).toBe(300);
299
300 yield* executor.executionHistory.recordMany("org", entries("failed"));
301 const failed = yield* executor.executionHistory.count({
302 where: { runId: "run-batched", status: "failed" },
303 });
304 expect(failed).toBe(300);
305 const totalAfterOverwrite = yield* executor.executionHistory.count({
306 where: { runId: "run-batched" },
307 });
308 expect(totalAfterOverwrite).toBe(300);
309 }),
310 );
311
312 it.effect("rolls back every plugin storage row when a bulk write fails", () =>
313 Effect.gen(function* () {
314 const config = makeTestConfig({
315 backend: "sqlite",
316 plugins: [executionHistoryPlugin] as const,
317 });
318 const executor = yield* Effect.acquireRelease(
319 createExecutor({
320 ...config,
321 db: failPluginStorageBulkWriteAfterFirstRow(config.db),
322 }),
323 (instance) =>
324 instance
325 .close()
326 .pipe(
327 Effect.ignore,
328 Effect.andThen(Effect.promise(() => config.testDb.close()).pipe(Effect.ignore)),
329 ),
330 );
331
332 const exit = yield* Effect.exit(
333 executor.executionHistory.recordMany("org", [
334 {
335 key: "call-first",
336 data: call({
337 runId: "run-rollback",
338 toolId: "browser",
339 status: "ok",
340 startedAt: "2026-05-29T12:00:00.000Z",
341 }),
342 },
343 {
344 key: "call-second",
345 data: call({
346 runId: "run-rollback",
347 toolId: "shell",
348 status: "ok",
349 startedAt: "2026-05-29T12:01:00.000Z",
350 }),
351 },
352 ]),
353 );
354 expect(Exit.isFailure(exit)).toBe(true);
355
356 const stored = yield* executor.executionHistory.query({
357 where: { runId: "run-rollback" },
358 });
359 expect(stored).toEqual([]);
360 }),
361 );
362
363 // The hazard this whole change exists to remove: the previous implementation
364 // deleted every target key and only then re-created the rows, so a failure
365 // between the two halves destroyed data the caller never meant to touch. An
366 // upsert inside a transaction cannot lose a row it did not successfully
367 // replace, so the ORIGINAL values must still be readable after the failure —
368 // not merely absent-and-consistent like the rolled-back insert above.
369 it.effect("leaves pre-existing rows intact when a bulk overwrite fails mid-batch", () =>
370 Effect.gen(function* () {
371 const config = makeTestConfig({
372 backend: "sqlite",
373 plugins: [executionHistoryPlugin] as const,
374 });
375 const executor = yield* Effect.acquireRelease(
376 createExecutor({
377 ...config,
378 db: failPluginStorageBulkWriteAfterFirstRow(config.db),
379 }),
380 (instance) =>
381 instance
382 .close()
383 .pipe(
384 Effect.ignore,
385 Effect.andThen(Effect.promise(() => config.testDb.close()).pipe(Effect.ignore)),
386 ),
387 );
388
389 // Seeded one row at a time, so the seeding itself never goes through the
390 // bulk path the fault injector breaks.
391 const original = [
392 {
393 key: "call-first",
394 data: call({
395 runId: "run-preexisting",
396 toolId: "browser",
397 status: "ok",
398 startedAt: "2026-05-29T12:00:00.000Z",
399 }),
400 },
401 {
402 key: "call-second",
403 data: call({
404 runId: "run-preexisting",
405 toolId: "shell",
406 status: "ok",
407 startedAt: "2026-05-29T12:01:00.000Z",
408 }),
409 },
410 ];
411 for (const row of original) {
412 yield* executor.executionHistory.record("org", row.key, row.data);
413 }
414
415 const exit = yield* Effect.exit(
416 executor.executionHistory.recordMany(
417 "org",
418 original.map((row) => ({
419 key: row.key,
420 data: { ...row.data, toolId: "overwritten", status: "failed" as const },
421 })),
422 ),
423 );
424 expect(Exit.isFailure(exit)).toBe(true);
425
426 const first = yield* executor.executionHistory.get("call-first");
427 const second = yield* executor.executionHistory.get("call-second");
428 expect(first?.data).toEqual(original[0]!.data);
429 expect(second?.data).toEqual(original[1]!.data);
430 }),
431 );
432
225 it.effect(433 it.effect(
226 "bulk puts large plugin storage row sets in bounded batches",434 "bulk puts large plugin storage row sets in bounded batches",
227 () =>435 () =>
packages/core/sdk/src/test-config.ts+9−0
@@ -53,6 +53,14 @@ const makeLazyTestFumaDb = (options: { · 52 unmodified lines
53 transaction: async (run) => (await start()).db.internal.transaction(run),53 transaction: async (run) => (await start()).db.internal.transaction(run),
54 updateMany: async (table, value) => (await start()).db.internal.updateMany(table, value),54 updateMany: async (table, value) => (await start()).db.internal.updateMany(table, value),
55 upsert: async (table, value) => (await start()).db.internal.upsert(table, value),55 upsert: async (table, value) => (await start()).db.internal.upsert(table, value),
56 upsertMany: async (table, value) => {
57 const actual = await start();
58 if (!actual.db.internal.upsertMany) {
59 // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: lazy test DB must expose the current FumaDB adapter surface
60 throw new Error("[FumaDB] upsertMany is not supported by this adapter.");
61 }
62 return actual.db.internal.upsertMany(table, value);
63 },
56 };64 };
57 65
58 const queryMethods = new Set<PropertyKey>([66 const queryMethods = new Set<PropertyKey>([
@@ -65,6 +73,7 @@ const makeLazyTestFumaDb = (options: { · 6 unmodified lines
65 "transaction",73 "transaction",
66 "updateMany",74 "updateMany",
67 "upsert",75 "upsert",
76 "upsertMany",
68 ]);77 ]);
69 78
70 const makeDb = (context?: ExecutorOwnerPolicyContext): FumaDb =>79 const makeDb = (context?: ExecutorOwnerPolicyContext): FumaDb =>

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t