Unwrap stale openapi output-schema envelopes at boot for libSQL apps

The libSQL-backed apps (local, selfhost) have no SQL migration chain at boot, so the envelope unwrap that cloud gets from drizzle 0002 ships as an idempotent startup data migration instead, following the runSqliteAuthConfigMigration pattern: plugin-openapi exports runSqliteOpenApiOutputSchemaMigration (structural envelope detection in JS, transactional rewrite, no-op on fresh databases and payload-shaped rows), and both boots call it next to the auth-config migration.

Rhys Sullivan committed 5 months ago

Commit fed31511 parent 18826555 files · +240 −0Browse files

Unified · Split

5 files

apps/host-selfhost/src/app.ts +6 −0
apps/local/src/executor.ts +11 −0
packages/plugins/openapi/src/sdk/index.ts +6 −0
packages/plugins/openapi/src/sdk/output-schema-migration.test.ts +103 −0
packages/plugins/openapi/src/sdk/output-schema-migration.ts +114 −0

apps/host-selfhost/src/app.ts+6−0

 import { composePluginApi, ExecutorApp, textFailureStrategy } from "@executor-js/api/server";
 import { runSqliteAuthConfigMigration } from "@executor-js/sdk/http-auth";
+import { runSqliteOpenApiOutputSchemaMigration } from "@executor-js/plugin-openapi";
 import { resolveAuthProviders } from "./auth";
 import { authConfigTransforms } from "./db/auth-config-migration";
   // canonical (same defensive-at-boot pattern as the column adds above).
   await runSqliteAuthConfigMigration(dbHandle.client, authConfigTransforms);
+  // One-off data migration: unwrap the retired {status, headers, data}
+  // transport envelope from persisted openapi tool output schemas (mirrors
+  // cloud's drizzle 0002). Idempotent — payload-shaped rows don't match.
+  await runSqliteOpenApiOutputSchemaMigration(dbHandle.client);
   // ---- auth providers ---------------------------------------------------
   // Better Auth: cookie/bearer/api-key identity + /api/auth handler + account
   // API + MCP OAuth seam, all over the shared libSQL handle.

apps/local/src/executor.ts+11−0

 import { Subject, Tenant, createExecutor, type AnyPlugin, type Executor } from "@executor-js/sdk";
 import { runSqliteAuthConfigMigration } from "@executor-js/sdk/http-auth";
+import { runSqliteOpenApiOutputSchemaMigration } from "@executor-js/plugin-openapi";
 import { collectTables } from "@executor-js/api/server";
 import { loadPluginsFromJsonc } from "@executor-js/config";
           new LocalExecutorCreateError({ message: CREATE_SQLITE_ERROR_MESSAGE, cause }),
       });
+      // One-off data migration: unwrap the retired {status, headers, data}
+      // transport envelope from persisted openapi tool output schemas
+      // (mirrors cloud's drizzle 0002). Idempotent — payload-shaped rows
+      // don't match the envelope signature.
+      yield* Effect.tryPromise({
+        try: () => runSqliteOpenApiOutputSchemaMigration(sqlite.client),
+        catch: (cause) =>
+          new LocalExecutorCreateError({ message: CREATE_SQLITE_ERROR_MESSAGE, cause }),
+      });
       // webBaseUrl is where the executor's web UI listens — same port as the
       // daemon API since the daemon serves both. Mirrors serve.ts's port
       // resolution so a custom $PORT flows through. EXECUTOR_WEB_BASE_URL

packages/plugins/openapi/src/sdk/index.ts+6−0

@@ -77,3 +77,9 @@ export { · 76 unmodified lines
77	export { variable, type ApiKeyAuthTemplate } from "@executor-js/sdk/http-auth";	77	export { variable, type ApiKeyAuthTemplate } from "@executor-js/sdk/http-auth";
78		78
79	export { migrateOpenApiAuthConfig } from "./migrate-config";	79	export { migrateOpenApiAuthConfig } from "./migrate-config";
		80
		81	export {
		82	runSqliteOpenApiOutputSchemaMigration,
		83	unwrapOpenApiTransportEnvelope,
		84	type SqliteToolSchemaClient,
		85	} from "./output-schema-migration";

packages/plugins/openapi/src/sdk/output-schema-migration.test.ts+103−0

@@ -0,0 +1,103 @@
	1	import { describe, expect, it } from "@effect/vitest";
	2
	3	import {
	4	runSqliteOpenApiOutputSchemaMigration,
	5	unwrapOpenApiTransportEnvelope,
	6	type SqliteToolSchemaClient,
	7	} from "./output-schema-migration";
	8
	9	// The exact shape openApiTransportOutputSchema used to emit.
	10	const envelope = (dataSchema: unknown) => ({
	11	type: "object",
	12	additionalProperties: false,
	13	required: ["status", "headers", "data"],
	14	properties: {
	15	status: { type: "integer" },
	16	headers: {
	17	type: "object",
	18	additionalProperties: { type: "string" },
	19	},
	20	data: dataSchema,
	21	},
	22	});
	23
	24	describe("unwrapOpenApiTransportEnvelope", () => {
	25	it("unwraps the envelope to its data schema", () => {
	26	const payload = { type: "object", properties: { name: { type: "string" } } };
	27	expect(unwrapOpenApiTransportEnvelope(envelope(payload))).toEqual({ outputSchema: payload });
	28	});
	29
	30	it("maps the empty data schema to null (new producer persists no schema)", () => {
	31	expect(unwrapOpenApiTransportEnvelope(envelope({}))).toEqual({ outputSchema: null });
	32	});
	33
	34	it("leaves payload-shaped schemas untouched", () => {
	35	expect(unwrapOpenApiTransportEnvelope({ $ref: "#/$defs/single_response" })).toBeUndefined();
	36	expect(
	37	unwrapOpenApiTransportEnvelope({
	38	// A user API that happens to return {status, headers, data} but isn't
	39	// the envelope (different property schemas, no additionalProperties).
	40	type: "object",
	41	required: ["status", "headers", "data"],
	42	properties: { status: { type: "string" }, headers: {}, data: {} },
	43	}),
	44	).toBeUndefined();
	45	expect(unwrapOpenApiTransportEnvelope(null)).toBeUndefined();
	46	expect(unwrapOpenApiTransportEnvelope("[]")).toBeUndefined();
	47	});
	48	});
	49
	50	// A tiny scripted fake standing in for a libSQL client.
	51	const makeFakeClient = (rows: Record<string, unknown>[], options?: { noTable?: boolean }) => {
	52	const log: unknown[] = [];
	53	const client: SqliteToolSchemaClient = {
	54	execute: (stmt) => {
	55	log.push(stmt);
	56	if (typeof stmt === "string" && stmt.includes("sqlite_master")) {
	57	return Promise.resolve({ rows: options?.noTable ? [] : [{ name: "tool" }] });
	58	}
	59	if (typeof stmt === "string" && stmt.startsWith("SELECT row_id")) {
	60	return Promise.resolve({ rows });
	61	}
	62	return Promise.resolve({ rows: [] });
	63	},
	64	};
	65	return { client, log };
	66	};
	67
	68	describe("runSqliteOpenApiOutputSchemaMigration", () => {
	69	it("rewrites envelope rows in a transaction and reports the count", async () => {
	70	const payload = { type: "array", items: { type: "object" } };
	71	const { client, log } = makeFakeClient([
	72	{ row_id: "a", output_schema: JSON.stringify(envelope(payload)) },
	73	{ row_id: "b", output_schema: JSON.stringify(envelope({})) },
	74	{ row_id: "c", output_schema: JSON.stringify({ $ref: "#/$defs/already_payload" }) },
	75	{ row_id: "d", output_schema: "not json" },
	76	]);
	77	const count = await runSqliteOpenApiOutputSchemaMigration(client);
	78	expect(count).toBe(2);
	79	expect(log).toContainEqual("BEGIN");
	80	expect(log).toContainEqual({
	81	sql: "UPDATE tool SET output_schema = ? WHERE row_id = ?",
	82	args: [JSON.stringify(payload), "a"],
	83	});
	84	expect(log).toContainEqual({
	85	sql: "UPDATE tool SET output_schema = ? WHERE row_id = ?",
	86	args: [null, "b"],
	87	});
	88	expect(log).toContainEqual("COMMIT");
	89	});
	90
	91	it("no-ops when every row is already payload-shaped", async () => {
	92	const { client, log } = makeFakeClient([
	93	{ row_id: "a", output_schema: JSON.stringify({ $ref: "#/$defs/already_payload" }) },
	94	]);
	95	expect(await runSqliteOpenApiOutputSchemaMigration(client)).toBe(0);
	96	expect(log).not.toContainEqual("BEGIN");
	97	});
	98
	99	it("treats a missing tool table as nothing to migrate", async () => {
	100	const { client } = makeFakeClient([], { noTable: true });
	101	expect(await runSqliteOpenApiOutputSchemaMigration(client)).toBe(0);
	102	});
	103	});

packages/plugins/openapi/src/sdk/output-schema-migration.ts+114−0

@@ -0,0 +1,114 @@
	1	/* oxlint-disable executor/no-try-catch-or-throw, executor/no-json-parse -- boundary: one-shot data migration drives a raw SQL client (JSON text columns, transaction + rollback) */
	2	// ---------------------------------------------------------------------------
	3	// One-off boot migration: unwrap the retired {status, headers, data}
	4	// transport envelope from persisted OpenAPI tool output schemas. The runtime
	5	// returns the upstream payload as `data` (status/headers live in the
	6	// ToolResult `http` side channel), so persisted schemas must describe the
	7	// payload only — otherwise describe previews show an envelope invocations no
	8	// longer return. Mirrors the cloud drizzle migration
	9	// (apps/cloud/drizzle/0002_unwrap_openapi_output_envelope.sql) for the
	10	// libSQL-backed apps, which have no SQL migration chain at boot.
	11	//
	12	// Idempotent by construction: payload-shaped rows don't match the envelope
	13	// signature, so re-running plans zero updates.
	14	// ---------------------------------------------------------------------------
	15
	16	const isRecord = (value: unknown): value is Record<string, unknown> =>
	17	typeof value === "object" && value !== null && !Array.isArray(value);
	18
	19	// `{"type": "integer"}` — the envelope's `status` property schema, exactly.
	20	const isEnvelopeStatusSchema = (value: unknown): boolean =>
	21	isRecord(value) && Object.keys(value).length === 1 && value.type === "integer";
	22
	23	// `{"type": "object", "additionalProperties": {"type": "string"}}` — the
	24	// envelope's `headers` property schema, exactly.
	25	const isEnvelopeHeadersSchema = (value: unknown): boolean =>
	26	isRecord(value) &&
	27	Object.keys(value).length === 2 &&
	28	value.type === "object" &&
	29	isRecord(value.additionalProperties) &&
	30	Object.keys(value.additionalProperties).length === 1 &&
	31	value.additionalProperties.type === "string";
	32
	33	/**
	34	* If `schema` is the retired transport envelope, return the payload schema
	35	* to persist instead (`null` when the envelope carried an empty `{}` data
	36	* schema — the new producer persists no output schema for those). Returns
	37	* undefined when the schema is not an envelope and the row must be left
	38	* untouched.
	39	*/
	40	export const unwrapOpenApiTransportEnvelope = (
	41	schema: unknown,
	42	): { readonly outputSchema: unknown \| null } \| undefined => {
	43	if (!isRecord(schema)) return undefined;
	44	if (schema.type !== "object" \|\| schema.additionalProperties !== false) return undefined;
	45	const required = schema.required;
	46	if (!Array.isArray(required) \|\| required.length !== 3) return undefined;
	47	if (!["status", "headers", "data"].every((key) => required.includes(key))) return undefined;
	48	const properties = schema.properties;
	49	if (!isRecord(properties) \|\| !("data" in properties)) return undefined;
	50	if (!isEnvelopeStatusSchema(properties.status)) return undefined;
	51	if (!isEnvelopeHeadersSchema(properties.headers)) return undefined;
	52	const data = properties.data;
	53	const outputSchema = isRecord(data) && Object.keys(data).length === 0 ? null : data;
	54	return { outputSchema };
	55	};
	56
	57	// ---------------------------------------------------------------------------
	58	// SQLite runner — shared by the libSQL-backed apps (local boot, selfhost
	59	// boot). Structural client interface so the plugin doesn't take a driver
	60	// dependency; `@libsql/client` satisfies it.
	61	// ---------------------------------------------------------------------------
	62
	63	export interface SqliteToolSchemaClient {
	64	execute(
	65	stmt: string \| { readonly sql: string; readonly args: readonly unknown[] },
	66	): Promise<{ readonly rows: readonly Record<string, unknown>[] }>;
	67	}
	68
	69	/** Unwrap envelope-shaped openapi tool output schemas in a SQLite database.
	70	* Idempotent; returns the number of rows rewritten. The `tool` table may
	71	* not exist yet on a fresh database — that counts as nothing to migrate. */
	72	export const runSqliteOpenApiOutputSchemaMigration = async (
	73	client: SqliteToolSchemaClient,
	74	): Promise<number> => {
	75	const exists = await client.execute(
	76	"SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'tool'",
	77	);
	78	if (exists.rows.length === 0) return 0;
	79
	80	const result = await client.execute(
	81	"SELECT row_id, output_schema FROM tool WHERE plugin_id = 'openapi' AND output_schema IS NOT NULL",
	82	);
	83	const updates: { readonly rowId: string; readonly outputSchema: unknown \| null }[] = [];
	84	for (const row of result.rows) {
	85	if (typeof row.row_id !== "string" \|\| typeof row.output_schema !== "string") continue;
	86	let schema: unknown;
	87	try {
	88	schema = JSON.parse(row.output_schema);
	89	} catch {
	90	continue;
	91	}
	92	const unwrapped = unwrapOpenApiTransportEnvelope(schema);
	93	if (unwrapped !== undefined) updates.push({ rowId: row.row_id, ...unwrapped });
	94	}
	95	if (updates.length === 0) return 0;
	96
	97	await client.execute("BEGIN");
	98	try {
	99	for (const update of updates) {
	100	await client.execute({
	101	sql: "UPDATE tool SET output_schema = ? WHERE row_id = ?",
	102	args: [
	103	update.outputSchema === null ? null : JSON.stringify(update.outputSchema),
	104	update.rowId,
	105	],
	106	});
	107	}
	108	await client.execute("COMMIT");
	109	} catch (cause) {
	110	await client.execute("ROLLBACK");
	111	throw cause;
	112	}
	113	return updates.length;
	114	};

Unwrap stale openapi output-schema envelopes at boot for libSQL apps

Keyboard shortcuts

General

Issues and pull requests

Issues, pull requests and Inbox

Repository code