fix(storage): require createMany on CustomAdapter; drizzle issues chunked bulk INSERT
Unified · Split
| @@ -207,6 +207,18 @@ export interface CustomAdapter { · 206 unmodified lines | |||
| 207 | select?: string[] | undefined; | 207 | select?: string[] | undefined; |
| 208 | }) => Effect.Effect<T, Error>; | 208 | }) => Effect.Effect<T, Error>; |
| 209 | 209 | ||
| 210 | /** | ||
| 211 | * Native bulk insert. Required because over a real network (e.g. | ||
| 212 | * Hyperdrive), per-row `create` inflates to N round-trips and can't | ||
| 213 | * finish inside a request budget for specs with ~1000+ rows. SQL | ||
| 214 | * backends issue a single multi-row INSERT; in-memory backends just | ||
| 215 | * push the rows. | ||
| 216 | */ | ||
| 217 | createMany: <T extends Record<string, unknown>>(data: { | ||
| 218 | model: string; | ||
| 219 | data: ReadonlyArray<T>; | ||
| 220 | }) => Effect.Effect<T[], Error>; | ||
| 221 | |||
| 210 | update: <T>(data: { | 222 | update: <T>(data: { |
| 211 | model: string; | 223 | model: string; |
| 212 | where: CleanedWhere[]; | 224 | where: CleanedWhere[]; |
| @@ -637,14 +637,36 @@ export const createAdapter = ( · 636 unmodified lines | |||
| 637 | forceAllowId?: boolean | undefined; | 637 | forceAllowId?: boolean | undefined; |
| 638 | }) => | 638 | }) => |
| 639 | Effect.gen(function* () { | 639 | Effect.gen(function* () { |
| 640 | const out: R[] = []; | 640 | // Delegates straight to the backend's native bulk insert — no |
| 641 | // per-row fallback, because over a real network connection | ||
| 642 | // (Hyperdrive, etc.) N round-trips would blow the request | ||
| 643 | // budget for specs with thousands of operations. Transforms | ||
| 644 | // still run per-row so JSON / dates / booleans serialize the | ||
| 645 | // same as single `create`. | ||
| 646 | const inputs: Record<string, unknown>[] = []; | ||
| 641 | for (const row of data.data) { | 647 | for (const row of data.data) { |
| 642 | const created = yield* self.create<T, R>({ | 648 | inputs.push( |
| 643 | model: data.model, | 649 | yield* maybeTransformInput( |
| 644 | data: row, | 650 | data.model, |
| 645 | forceAllowId: data.forceAllowId, | 651 | row as Record<string, unknown>, |
| 646 | }); | 652 | "create", |
| 647 | out.push(created); | 653 | data.forceAllowId === true, |
| 654 | ), | ||
| 655 | ); | ||
| 656 | } | ||
| 657 | const res = yield* inner.createMany({ | ||
| 658 | model: getModelName(data.model), | ||
| 659 | data: inputs, | ||
| 660 | }); | ||
| 661 | const out: R[] = []; | ||
| 662 | for (const row of res) { | ||
| 663 | out.push( | ||
| 664 | (yield* maybeTransformOutput( | ||
| 665 | data.model, | ||
| 666 | row as Record<string, unknown>, | ||
| 667 | undefined, | ||
| 668 | )) as unknown as R, | ||
| 669 | ); | ||
| 648 | } | 670 | } |
| 649 | return out as unknown as readonly R[]; | 671 | return out as unknown as readonly R[]; |
| 650 | }), | 672 | }), |
| @@ -168,6 +168,13 @@ export const makeMemoryAdapter = ( · 167 unmodified lines | |||
| 168 | return data; | 168 | return data; |
| 169 | }), | 169 | }), |
| 170 | 170 | ||
| 171 | createMany: ({ model, data }) => | ||
| 172 | Effect.sync(() => { | ||
| 173 | const table = tableFor(model); | ||
| 174 | for (const row of data) table.push(row as Row); | ||
| 175 | return data.slice() as never; | ||
| 176 | }), | ||
| 177 | |||
| 171 | findOne: ({ model, where, join }) => | 178 | findOne: ({ model, where, join }) => |
| 172 | Effect.sync(() => { | 179 | Effect.sync(() => { |
| 173 | const rows = filterWhere(tableFor(model), where); | 180 | const rows = filterWhere(tableFor(model), where); |
| @@ -312,6 +312,30 @@ export const drizzleAdapter = (options: DrizzleAdapterOptions): DBAdapter => { · 311 unmodified lines | |||
| 312 | return row as never; | 312 | return row as never; |
| 313 | }), | 313 | }), |
| 314 | 314 | ||
| 315 | // Real multi-row INSERT in fixed-size chunks. One statement per | ||
| 316 | // chunk, not one per row — per-row loops blow the Hyperdrive | ||
| 317 | // request budget on specs with thousands of operations. Chunking | ||
| 318 | // (vs a single giant statement) also keeps payload size bounded: | ||
| 319 | // JSON columns like tool schemas / operation bindings can be a | ||
| 320 | // few KB each, so a 2700-row insert becomes a >10MB statement | ||
| 321 | // otherwise, which chokes both Hyperdrive ingress and WASM | ||
| 322 | // Postgres (PGlite) in the test harness. | ||
| 323 | createMany: ({ model, data }) => | ||
| 324 | Effect.gen(function* () { | ||
| 325 | if (data.length === 0) return [] as never; | ||
| 326 | const table = getTable(model); | ||
| 327 | const CHUNK = 500; | ||
| 328 | const all: Record<string, unknown>[] = []; | ||
| 329 | for (let i = 0; i < data.length; i += CHUNK) { | ||
| 330 | const slice = data.slice(i, i + CHUNK) as Record<string, unknown>[]; | ||
| 331 | const rows = (yield* runPromise("insert many returning", () => | ||
| 332 | db.insert(table).values(slice).returning(), | ||
| 333 | )) as Record<string, unknown>[]; | ||
| 334 | for (const row of rows) all.push(row); | ||
| 335 | } | ||
| 336 | return all as never; | ||
| 337 | }), | ||
| 338 | |||
| 315 | findOne: ({ model, where, join }) => | 339 | findOne: ({ model, where, join }) => |
| 316 | Effect.gen(function* () { | 340 | Effect.gen(function* () { |
| 317 | const table = getTable(model); | 341 | const table = getTable(model); |