History

fix storage + rm upgrade flow

Rhys Sullivan committed 8 months ago

Commit 036371c1 parent c8ece816 files · +11 −388Browse files

Unified · Split

6 files
  1. executor/apps/web/src/components/tools/add/source-dialog.tsx +0 −60
  2. executor/apps/web/src/components/tools/explorer/source-form-panel.tsx +0 −60
  3. executor/packages/database/convex/executorNode.ts +0 −28
  4. executor/packages/database/convex/http/mcp_auth.ts +4 −39
  5. executor/packages/database/src/runtime/tool_upgrade.ts +0 −185
  6. executor/packages/runner-sandbox-host/src/index.ts +7 −16
executor/apps/web/src/components/tools/add/source-dialog.tsx+0−60
@@ -38,7 +38,6 @@ import { · 37 unmodified lines
3838 CatalogViewSection,
3939 CustomViewSection,
4040 } from "./source/dialog-sections";
41−import { createCustomSourceConfig } from "./source/dialog-helpers";
4241 import { normalizeSourceEndpoint } from "@/lib/tools/source-url";
4342 import { SourceAuthPanel } from "./source/auth-panel";
4443 import {
@@ -87,7 +86,6 @@ export function AddSourceDialog({ · 42 unmodified lines
8786 const { context } = useSession();
8887 const upsertToolSource = useAction(convexApi.workspace.upsertToolSource);
8988 const deleteToolSource = useMutation(convexApi.workspace.deleteToolSource);
90− const previewOpenApiSourceUpgrade = useAction(convexApi.executorNode.previewOpenApiSourceUpgrade);
9189 const credentials = useQuery(
9290 convexApi.workspace.listCredentials,
9391 workspaceQueryArgs(context),
@@ -138,64 +136,6 @@ export function AddSourceDialog({ · 44 unmodified lines
138136 return;
139137 }
140138
141− if (sourceToEdit?.type === "openapi" && form.type === "openapi") {
142− const previewConfig = createCustomSourceConfig({
143− type: form.type,
144− endpoint: form.endpoint.trim(),
145− baseUrl: form.baseUrl,
146− auth: form.buildAuthConfig(),
147− useCredentialedFetch: form.useCredentialedFetch,
148− mcpTransport: form.mcpTransport,
149− accountId: context.accountId,
150− });
151−
152− const previewResult = await Result.tryPromise(() =>
153− previewOpenApiSourceUpgrade({
154− workspaceId: context.workspaceId,
155− sessionId: context.sessionId,
156− sourceId: sourceToEdit.id,
157− name: form.name.trim(),
158− config: previewConfig,
159− }),
160− );
161−
162− if (!previewResult.isOk()) {
163− toast.error(resultErrorMessage(previewResult.error, "Failed to preview OpenAPI changes"));
164− return;
165− }
166−
167− const preview = previewResult.value;
168− if (preview.hasChanges) {
169− const sample = (items: string[]) => items.slice(0, 8).join("\n- ");
170− const added = sample(preview.addedPaths);
171− const removed = sample(preview.removedPaths);
172− const changed = sample(preview.changedPaths);
173−
174− const message = [
175− `OpenAPI upgrade for \"${preview.currentSourceName}\"`,
176− "",
177− `Added: ${preview.addedCount}`,
178− `Removed: ${preview.removedCount}`,
179− `Changed: ${preview.changedCount}`,
180− "",
181− added ? `Added sample:\n- ${added}` : "",
182− removed ? `Removed sample:\n- ${removed}` : "",
183− changed ? `Changed sample:\n- ${changed}` : "",
184− preview.truncated ? "\n(Preview truncated)" : "",
185− "",
186− "Apply this upgrade now?",
187− ]
188− .filter((line) => line.length > 0)
189− .join("\n");
190−
191− const approved = window.confirm(message);
192− if (!approved) {
193− toast.message("Upgrade cancelled");
194− return;
195− }
196− }
197− }
198−
199139 if (form.isNameTaken(form.name)) {
200140 toast.error(`Source name "${form.name.trim()}" already exists`);
201141 return;
executor/apps/web/src/components/tools/explorer/source-form-panel.tsx+0−60
@@ -30,7 +30,6 @@ import { SourceQualitySummary } from "../source/quality-details"; · 29 unmodified lines
3030 import { useSession } from "@/lib/session-context";
3131 import { convexApi } from "@/lib/convex-api";
3232 import { normalizeSourceEndpoint } from "@/lib/tools/source-url";
33−import { createCustomSourceConfig } from "../add/source/dialog-helpers";
3433 import {
3534 CatalogViewSection,
3635 CustomViewSection,
@@ -131,7 +130,6 @@ export function SourceFormPanel({ · 94 unmodified lines
131130 const { context } = useSession();
132131 const upsertToolSource = useAction(convexApi.workspace.upsertToolSource);
133132 const deleteToolSource = useMutation(convexApi.workspace.deleteToolSource);
134− const previewOpenApiSourceUpgrade = useAction(convexApi.executorNode.previewOpenApiSourceUpgrade);
135133 const credentials = useQuery(
136134 convexApi.workspace.listCredentials,
137135 workspaceQueryArgs(context),
@@ -176,64 +174,6 @@ export function SourceFormPanel({ · 38 unmodified lines
176174 return;
177175 }
178176
179− if (sourceToEdit?.type === "openapi" && form.type === "openapi") {
180− const previewConfig = createCustomSourceConfig({
181− type: form.type,
182− endpoint: form.endpoint.trim(),
183− baseUrl: form.baseUrl,
184− auth: form.buildAuthConfig(),
185− useCredentialedFetch: form.useCredentialedFetch,
186− mcpTransport: form.mcpTransport,
187− accountId: context.accountId,
188− });
189−
190− const previewResult = await Result.tryPromise(() =>
191− previewOpenApiSourceUpgrade({
192− workspaceId: context.workspaceId,
193− sessionId: context.sessionId,
194− sourceId: sourceToEdit.id,
195− name: form.name.trim(),
196− config: previewConfig,
197− }),
198− );
199−
200− if (!previewResult.isOk()) {
201− toast.error(resultErrorMessage(previewResult.error, "Failed to preview OpenAPI changes"));
202− return;
203− }
204−
205− const preview = previewResult.value;
206− if (preview.hasChanges) {
207− const sample = (items: string[]) => items.slice(0, 8).join("\n- ");
208− const added = sample(preview.addedPaths);
209− const removed = sample(preview.removedPaths);
210− const changed = sample(preview.changedPaths);
211−
212− const message = [
213− `OpenAPI upgrade for \"${preview.currentSourceName}\"`,
214− "",
215− `Added: ${preview.addedCount}`,
216− `Removed: ${preview.removedCount}`,
217− `Changed: ${preview.changedCount}`,
218− "",
219− added ? `Added sample:\n- ${added}` : "",
220− removed ? `Removed sample:\n- ${removed}` : "",
221− changed ? `Changed sample:\n- ${changed}` : "",
222− preview.truncated ? "\n(Preview truncated)" : "",
223− "",
224− "Apply this upgrade now?",
225− ]
226− .filter((line) => line.length > 0)
227− .join("\n");
228−
229− const approved = window.confirm(message);
230− if (!approved) {
231− toast.message("Upgrade cancelled");
232− return;
233− }
234− }
235− }
236−
237177 if (form.isNameTaken(form.name)) {
238178 toast.error(`Source name "${form.name.trim()}" already exists`);
239179 return;
executor/packages/database/convex/executorNode.ts+0−28
@@ -22,7 +22,6 @@ import { handleExternalToolCallRequest } from "../src/runtime/external_tool_call · 21 unmodified lines
2222 import { jsonObjectValidator } from "../src/database/validators";
2323 import { workspaceAction, type WorkspaceActionContext } from "../../core/src/function-builders";
2424 import { encodeToolCallResultForTransport } from "../../core/src/tool-call-result-transport";
25−import { previewOpenApiSourceUpgradeForContext, type OpenApiUpgradeDiffPreview } from "../src/runtime/tool_upgrade";
2625 import { getStorageProvider, type StorageEncoding, type StorageProvider } from "../src/runtime/storage_provider";
2726 import { shouldTouchStorageOnRead } from "../src/runtime/storage_touch_policy";
2827 import { shouldRefreshStorageUsage } from "../src/runtime/storage_usage_refresh";
@@ -210,33 +209,6 @@ export const rebuildToolInventoryInternal = internalAction({ · 181 unmodified lines
210209 },
211210 });
212211
213−export const previewOpenApiSourceUpgrade = workspaceAction({
214− method: "POST",
215− args: {
216− accountId: v.optional(vv.id("accounts")),
217− sourceId: v.string(),
218− name: v.string(),
219− config: jsonObjectValidator,
220− },
221− handler: async (ctx, args): Promise<OpenApiUpgradeDiffPreview> => {
222− assertMatchesCanonicalAccountId(args.accountId, ctx.accountId);
223−
224− return await previewOpenApiSourceUpgradeForContext(
225− ctx,
226− {
227− workspaceId: ctx.workspaceId,
228− accountId: ctx.accountId,
229− clientId: ctx.clientId,
230− },
231− {
232− sourceId: args.sourceId,
233− name: args.name,
234− config: args.config,
235− },
236− );
237− },
238−});
239−
240212 export const storageListDirectory = workspaceAction({
241213 method: "POST",
242214 args: {
executor/packages/database/convex/http/mcp_auth.ts+4−39
@@ -38,29 +38,9 @@ function mcpAuthRequired(): boolean { · 37 unmodified lines
3838 return isTruthyEnvValue(process.env.EXECUTOR_ENFORCE_MCP_AUTH);
3939 }
4040
41−function configuredMcpAudiences(): string[] {
42− const raw = (process.env.MCP_AUTH_AUDIENCE ?? "").trim();
43− if (!raw) {
44− return [];
45− }
46−
47− return raw
48− .split(",")
49− .map((entry) => entry.trim())
50− .filter((entry) => entry.length > 0);
51−}
52−
53−function deriveMcpAudiences(request: Request): string[] {
54− const url = new URL(request.url);
55− const derived = [
56− new URL(MCP_PATH, url.origin).toString(),
57− new URL(LEGACY_MCP_PATH, url.origin).toString(),
58− ];
59−
60− const configured = configuredMcpAudiences();
61− const merged = [...configured, ...derived];
62− return Array.from(new Set(merged));
63−}
41+// WorkOS AuthKit tokens use the requesting MCP client's client ID as the
42+// audience, which varies per client. Per WorkOS docs, only issuer + signature
43+// verification is required — audience checks are intentionally omitted.
6444
6545 function parseWorkspaceId(raw: string): Id<"workspaces"> {
6646 return raw as Id<"workspaces">;
@@ -165,7 +145,6 @@ export async function verifyMcpToken( · 98 unmodified lines
165145 try {
166146 const { payload } = await jwtVerify(token, config.jwks, {
167147 issuer: config.authorizationServer,
168− audience: deriveMcpAudiences(request),
169148 });
170149 if (typeof payload.sub === "string" && payload.sub.length > 0) {
171150 const providerClaim = typeof payload.provider === "string" ? payload.provider : undefined;
@@ -177,21 +156,7 @@ export async function verifyMcpToken( · 5 unmodified lines
177156 }
178157 }
179158 } catch (error) {
180− // Decode the token payload (without verification) to log what we received vs expected
181− try {
182− const [, payloadB64] = token.split(".");
183− if (payloadB64) {
184− const decoded = JSON.parse(atob(payloadB64.replace(/-/g, "+").replace(/_/g, "/")));
185− console.error("MCP token verification failed:", error instanceof Error ? error.message : String(error), {
186− tokenIssuer: decoded.iss,
187− tokenAudience: decoded.aud,
188− expectedIssuer: config.authorizationServer,
189− expectedAudiences: deriveMcpAudiences(request),
190− });
191− }
192− } catch {
193− console.error("MCP token verification failed:", error instanceof Error ? error.message : String(error));
194− }
159+ console.error("MCP token verification failed:", error instanceof Error ? error.message : String(error));
195160 }
196161 }
197162
executor/packages/database/src/runtime/tool_upgrade.ts+0−185
@@ -1,185 +0,0 @@
1−import type { ActionCtx } from "../../convex/_generated/server";
2−import { internal } from "../../convex/_generated/api";
3−import type { Id } from "../../convex/_generated/dataModel.d.ts";
4−import type { ToolSourceRecord } from "../../../core/src/types";
5−import { normalizeExternalToolSource, loadSourceArtifact } from "./tool_source_loading";
6−
7−export interface OpenApiUpgradeDiffPreview {
8− sourceId: string;
9− currentSourceName: string;
10− proposedSourceName: string;
11− currentToolCount: number;
12− proposedToolCount: number;
13− addedCount: number;
14− removedCount: number;
15− changedCount: number;
16− hasChanges: boolean;
17− addedPaths: string[];
18− removedPaths: string[];
19− changedPaths: string[];
20− truncated: boolean;
21− warnings: string[];
22−}
23−
24−const PREVIEW_LIST_LIMIT = 200;
25−
26−function clampPreview(items: string[]): { items: string[]; truncated: boolean } {
27− if (items.length <= PREVIEW_LIST_LIMIT) {
28− return { items, truncated: false };
29− }
30−
31− return {
32− items: items.slice(0, PREVIEW_LIST_LIMIT),
33− truncated: true,
34− };
35−}
36−
37−async function listCurrentSourceToolMap(
38− ctx: ActionCtx,
39− workspaceId: Id<"workspaces">,
40− sourceLabel: string,
41−): Promise<Map<string, string>> {
42− const tools = new Map<string, string>();
43− let cursor: string | undefined;
44−
45− while (true) {
46− const page: {
47− continueCursor: string | null;
48− items: Array<{ path: string }>;
49− } = await ctx.runQuery(internal.toolRegistry.listToolsBySourcePage, {
50− workspaceId,
51− source: sourceLabel,
52− cursor,
53− limit: 500,
54− });
55−
56− const payloads: Array<{ path: string; serializedToolJson: string }> = await ctx.runQuery(
57− internal.toolRegistry.getSerializedToolsByPaths,
58− {
59− workspaceId,
60− paths: page.items.map((item) => item.path),
61− },
62− );
63−
64− for (const payload of payloads) {
65− tools.set(payload.path, payload.serializedToolJson);
66− }
67−
68− if (page.continueCursor === null) {
69− break;
70− }
71−
72− cursor = page.continueCursor;
73− }
74−
75− return tools;
76−}
77−
78−export async function previewOpenApiSourceUpgradeForContext(
79− ctx: ActionCtx,
80− context: { workspaceId: Id<"workspaces">; accountId?: Id<"accounts">; clientId?: string },
81− args: { sourceId: string; name: string; config: Record<string, unknown> },
82−): Promise<OpenApiUpgradeDiffPreview> {
83− const sources: ToolSourceRecord[] = await ctx.runQuery(internal.database.listToolSources, {
84− workspaceId: context.workspaceId,
85− });
86− const existing = sources.find((source) => source.id === args.sourceId);
87− if (!existing) {
88− throw new Error("Source not found");
89− }
90− if (existing.type !== "openapi") {
91− throw new Error("Upgrade preview is only supported for OpenAPI sources");
92− }
93−
94− const candidateName = args.name.trim();
95− if (!candidateName) {
96− throw new Error("Source name is required");
97− }
98−
99− const candidateSource: ToolSourceRecord = {
100− ...existing,
101− name: candidateName,
102− type: "openapi",
103− config: args.config,
104− enabled: true,
105− updatedAt: Date.now(),
106− };
107−
108− const normalized = normalizeExternalToolSource(candidateSource);
109− if (normalized.isErr()) {
110− throw new Error(normalized.error.message);
111− }
112−
113− if (normalized.value.type !== "openapi") {
114− throw new Error("Upgrade preview expects an OpenAPI source");
115− }
116−
117− const loaded = await loadSourceArtifact(ctx, normalized.value, {
118− includeDts: false,
119− workspaceId: context.workspaceId,
120− accountId: context.accountId,
121− });
122−
123− const proposedSerializedTools = loaded.artifact?.tools ?? [];
124− if (!loaded.artifact) {
125− const reason = loaded.warnings[0] ?? "No tools generated from proposed spec";
126− throw new Error(reason);
127− }
128−
129− const proposedByPath = new Map<string, string>();
130− for (const tool of proposedSerializedTools) {
131− proposedByPath.set(tool.path, JSON.stringify(tool));
132− }
133−
134− const currentByPath = await listCurrentSourceToolMap(
135− ctx,
136− context.workspaceId,
137− `openapi:${existing.name}`,
138− );
139−
140− const added: string[] = [];
141− const removed: string[] = [];
142− const changed: string[] = [];
143−
144− for (const [path, serialized] of proposedByPath.entries()) {
145− const current = currentByPath.get(path);
146− if (!current) {
147− added.push(path);
148− continue;
149− }
150− if (current !== serialized) {
151− changed.push(path);
152− }
153− }
154−
155− for (const path of currentByPath.keys()) {
156− if (!proposedByPath.has(path)) {
157− removed.push(path);
158− }
159− }
160−
161− added.sort((a, b) => a.localeCompare(b));
162− removed.sort((a, b) => a.localeCompare(b));
163− changed.sort((a, b) => a.localeCompare(b));
164−
165− const cappedAdded = clampPreview(added);
166− const cappedRemoved = clampPreview(removed);
167− const cappedChanged = clampPreview(changed);
168−
169− return {
170− sourceId: existing.id,
171− currentSourceName: existing.name,
172− proposedSourceName: candidateName,
173− currentToolCount: currentByPath.size,
174− proposedToolCount: proposedByPath.size,
175− addedCount: added.length,
176− removedCount: removed.length,
177− changedCount: changed.length,
178− hasChanges: added.length > 0 || removed.length > 0 || changed.length > 0,
179− addedPaths: cappedAdded.items,
180− removedPaths: cappedRemoved.items,
181− changedPaths: cappedChanged.items,
182− truncated: cappedAdded.truncated || cappedRemoved.truncated || cappedChanged.truncated,
183− warnings: loaded.warnings,
184− };
185−}
executor/packages/runner-sandbox-host/src/index.ts+7−16
@@ -148,20 +148,9 @@ export class AgentFSStorageObject extends DurableObject<Env> { · 147 unmodified lines
148148 throw new Error("sqlite.query rejects multi-statement SQL payloads");
149149 }
150150
151− if (mode === "read") {
152− this.ctx.storage.sql.exec("PRAGMA query_only = 1");
153− try {
154− const cursor = this.ctx.storage.sql.exec(sql, ...params);
155− const rows = cursor.toArray().slice(0, Math.max(1, Math.floor(maxRows))) as Record<string, unknown>[];
156− return {
157− mode,
158− rows,
159− rowCount: rows.length,
160− };
161− } finally {
162− this.ctx.storage.sql.exec("PRAGMA query_only = 0");
163− }
164− }
151+ // Note: We rely on the application-level isReadOnlySql() check above
152+ // instead of PRAGMA query_only, because Cloudflare Durable Object SQLite
153+ // restricts PRAGMA usage and throws SQLITE_AUTH when setting query_only.
165154
166155 const cursor = this.ctx.storage.sql.exec(sql, ...params);
167156 if (mode === "write") {
@@ -172,10 +161,12 @@ export class AgentFSStorageObject extends DurableObject<Env> { · 4 unmodified lines
172161 };
173162 }
174163
164+ // read mode: return rows
165+ const rows = cursor.toArray().slice(0, Math.max(1, Math.floor(maxRows))) as Record<string, unknown>[];
175166 return {
176167 mode,
177− rows: [],
178− rowCount: 0,
168+ rows,
169+ rowCount: rows.length,
179170 };
180171 }
181172