History

fix storage + rm upgrade flow

Rhys Sullivan committed 8 months ago

Commit 036371c1 parent c8ece816 files · +11 −388Browse files

Unified · Split

6 files
  1. executor/apps/web/src/components/tools/add/source-dialog.tsx +0 −60
  2. executor/apps/web/src/components/tools/explorer/source-form-panel.tsx +0 −60
  3. executor/packages/database/convex/executorNode.ts +0 −28
  4. executor/packages/database/convex/http/mcp_auth.ts +4 −39
  5. executor/packages/database/src/runtime/tool_upgrade.ts +0 −185
  6. executor/packages/runner-sandbox-host/src/index.ts +7 −16
executor/apps/web/src/components/tools/add/source-dialog.tsx+0−60
@@ -38,7 +38,6 @@ import { · 37 unmodified lines
38 CatalogViewSection,38 CatalogViewSection,
39 CustomViewSection,39 CustomViewSection,
40} from "./source/dialog-sections";40} from "./source/dialog-sections";
41import { createCustomSourceConfig } from "./source/dialog-helpers";
42import { normalizeSourceEndpoint } from "@/lib/tools/source-url";41import { normalizeSourceEndpoint } from "@/lib/tools/source-url";
43import { SourceAuthPanel } from "./source/auth-panel";42import { SourceAuthPanel } from "./source/auth-panel";
44import {43import {
@@ -87,7 +86,6 @@ export function AddSourceDialog({ · 42 unmodified lines
87 const { context } = useSession();86 const { context } = useSession();
88 const upsertToolSource = useAction(convexApi.workspace.upsertToolSource);87 const upsertToolSource = useAction(convexApi.workspace.upsertToolSource);
89 const deleteToolSource = useMutation(convexApi.workspace.deleteToolSource);88 const deleteToolSource = useMutation(convexApi.workspace.deleteToolSource);
90 const previewOpenApiSourceUpgrade = useAction(convexApi.executorNode.previewOpenApiSourceUpgrade);
91 const credentials = useQuery(89 const credentials = useQuery(
92 convexApi.workspace.listCredentials,90 convexApi.workspace.listCredentials,
93 workspaceQueryArgs(context),91 workspaceQueryArgs(context),
@@ -138,64 +136,6 @@ export function AddSourceDialog({ · 44 unmodified lines
138 return;136 return;
139 }137 }
140 138
141 if (sourceToEdit?.type === "openapi" && form.type === "openapi") {
142 const previewConfig = createCustomSourceConfig({
143 type: form.type,
144 endpoint: form.endpoint.trim(),
145 baseUrl: form.baseUrl,
146 auth: form.buildAuthConfig(),
147 useCredentialedFetch: form.useCredentialedFetch,
148 mcpTransport: form.mcpTransport,
149 accountId: context.accountId,
150 });
151
152 const previewResult = await Result.tryPromise(() =>
153 previewOpenApiSourceUpgrade({
154 workspaceId: context.workspaceId,
155 sessionId: context.sessionId,
156 sourceId: sourceToEdit.id,
157 name: form.name.trim(),
158 config: previewConfig,
159 }),
160 );
161
162 if (!previewResult.isOk()) {
163 toast.error(resultErrorMessage(previewResult.error, "Failed to preview OpenAPI changes"));
164 return;
165 }
166
167 const preview = previewResult.value;
168 if (preview.hasChanges) {
169 const sample = (items: string[]) => items.slice(0, 8).join("\n- ");
170 const added = sample(preview.addedPaths);
171 const removed = sample(preview.removedPaths);
172 const changed = sample(preview.changedPaths);
173
174 const message = [
175 `OpenAPI upgrade for \"${preview.currentSourceName}\"`,
176 "",
177 `Added: ${preview.addedCount}`,
178 `Removed: ${preview.removedCount}`,
179 `Changed: ${preview.changedCount}`,
180 "",
181 added ? `Added sample:\n- ${added}` : "",
182 removed ? `Removed sample:\n- ${removed}` : "",
183 changed ? `Changed sample:\n- ${changed}` : "",
184 preview.truncated ? "\n(Preview truncated)" : "",
185 "",
186 "Apply this upgrade now?",
187 ]
188 .filter((line) => line.length > 0)
189 .join("\n");
190
191 const approved = window.confirm(message);
192 if (!approved) {
193 toast.message("Upgrade cancelled");
194 return;
195 }
196 }
197 }
198
199 if (form.isNameTaken(form.name)) {139 if (form.isNameTaken(form.name)) {
200 toast.error(`Source name "${form.name.trim()}" already exists`);140 toast.error(`Source name "${form.name.trim()}" already exists`);
201 return;141 return;
executor/apps/web/src/components/tools/explorer/source-form-panel.tsx+0−60
@@ -30,7 +30,6 @@ import { SourceQualitySummary } from "../source/quality-details"; · 29 unmodified lines
30import { useSession } from "@/lib/session-context";30import { useSession } from "@/lib/session-context";
31import { convexApi } from "@/lib/convex-api";31import { convexApi } from "@/lib/convex-api";
32import { normalizeSourceEndpoint } from "@/lib/tools/source-url";32import { normalizeSourceEndpoint } from "@/lib/tools/source-url";
33import { createCustomSourceConfig } from "../add/source/dialog-helpers";
34import {33import {
35 CatalogViewSection,34 CatalogViewSection,
36 CustomViewSection,35 CustomViewSection,
@@ -131,7 +130,6 @@ export function SourceFormPanel({ · 94 unmodified lines
131 const { context } = useSession();130 const { context } = useSession();
132 const upsertToolSource = useAction(convexApi.workspace.upsertToolSource);131 const upsertToolSource = useAction(convexApi.workspace.upsertToolSource);
133 const deleteToolSource = useMutation(convexApi.workspace.deleteToolSource);132 const deleteToolSource = useMutation(convexApi.workspace.deleteToolSource);
134 const previewOpenApiSourceUpgrade = useAction(convexApi.executorNode.previewOpenApiSourceUpgrade);
135 const credentials = useQuery(133 const credentials = useQuery(
136 convexApi.workspace.listCredentials,134 convexApi.workspace.listCredentials,
137 workspaceQueryArgs(context),135 workspaceQueryArgs(context),
@@ -176,64 +174,6 @@ export function SourceFormPanel({ · 38 unmodified lines
176 return;174 return;
177 }175 }
178 176
179 if (sourceToEdit?.type === "openapi" && form.type === "openapi") {
180 const previewConfig = createCustomSourceConfig({
181 type: form.type,
182 endpoint: form.endpoint.trim(),
183 baseUrl: form.baseUrl,
184 auth: form.buildAuthConfig(),
185 useCredentialedFetch: form.useCredentialedFetch,
186 mcpTransport: form.mcpTransport,
187 accountId: context.accountId,
188 });
189
190 const previewResult = await Result.tryPromise(() =>
191 previewOpenApiSourceUpgrade({
192 workspaceId: context.workspaceId,
193 sessionId: context.sessionId,
194 sourceId: sourceToEdit.id,
195 name: form.name.trim(),
196 config: previewConfig,
197 }),
198 );
199
200 if (!previewResult.isOk()) {
201 toast.error(resultErrorMessage(previewResult.error, "Failed to preview OpenAPI changes"));
202 return;
203 }
204
205 const preview = previewResult.value;
206 if (preview.hasChanges) {
207 const sample = (items: string[]) => items.slice(0, 8).join("\n- ");
208 const added = sample(preview.addedPaths);
209 const removed = sample(preview.removedPaths);
210 const changed = sample(preview.changedPaths);
211
212 const message = [
213 `OpenAPI upgrade for \"${preview.currentSourceName}\"`,
214 "",
215 `Added: ${preview.addedCount}`,
216 `Removed: ${preview.removedCount}`,
217 `Changed: ${preview.changedCount}`,
218 "",
219 added ? `Added sample:\n- ${added}` : "",
220 removed ? `Removed sample:\n- ${removed}` : "",
221 changed ? `Changed sample:\n- ${changed}` : "",
222 preview.truncated ? "\n(Preview truncated)" : "",
223 "",
224 "Apply this upgrade now?",
225 ]
226 .filter((line) => line.length > 0)
227 .join("\n");
228
229 const approved = window.confirm(message);
230 if (!approved) {
231 toast.message("Upgrade cancelled");
232 return;
233 }
234 }
235 }
236
237 if (form.isNameTaken(form.name)) {177 if (form.isNameTaken(form.name)) {
238 toast.error(`Source name "${form.name.trim()}" already exists`);178 toast.error(`Source name "${form.name.trim()}" already exists`);
239 return;179 return;
executor/packages/database/convex/executorNode.ts+0−28
@@ -22,7 +22,6 @@ import { handleExternalToolCallRequest } from "../src/runtime/external_tool_call · 21 unmodified lines
22import { jsonObjectValidator } from "../src/database/validators";22import { jsonObjectValidator } from "../src/database/validators";
23import { workspaceAction, type WorkspaceActionContext } from "../../core/src/function-builders";23import { workspaceAction, type WorkspaceActionContext } from "../../core/src/function-builders";
24import { encodeToolCallResultForTransport } from "../../core/src/tool-call-result-transport";24import { encodeToolCallResultForTransport } from "../../core/src/tool-call-result-transport";
25import { previewOpenApiSourceUpgradeForContext, type OpenApiUpgradeDiffPreview } from "../src/runtime/tool_upgrade";
26import { getStorageProvider, type StorageEncoding, type StorageProvider } from "../src/runtime/storage_provider";25import { getStorageProvider, type StorageEncoding, type StorageProvider } from "../src/runtime/storage_provider";
27import { shouldTouchStorageOnRead } from "../src/runtime/storage_touch_policy";26import { shouldTouchStorageOnRead } from "../src/runtime/storage_touch_policy";
28import { shouldRefreshStorageUsage } from "../src/runtime/storage_usage_refresh";27import { shouldRefreshStorageUsage } from "../src/runtime/storage_usage_refresh";
@@ -210,33 +209,6 @@ export const rebuildToolInventoryInternal = internalAction({ · 181 unmodified lines
210 },209 },
211});210});
212 211
213export const previewOpenApiSourceUpgrade = workspaceAction({
214 method: "POST",
215 args: {
216 accountId: v.optional(vv.id("accounts")),
217 sourceId: v.string(),
218 name: v.string(),
219 config: jsonObjectValidator,
220 },
221 handler: async (ctx, args): Promise<OpenApiUpgradeDiffPreview> => {
222 assertMatchesCanonicalAccountId(args.accountId, ctx.accountId);
223
224 return await previewOpenApiSourceUpgradeForContext(
225 ctx,
226 {
227 workspaceId: ctx.workspaceId,
228 accountId: ctx.accountId,
229 clientId: ctx.clientId,
230 },
231 {
232 sourceId: args.sourceId,
233 name: args.name,
234 config: args.config,
235 },
236 );
237 },
238});
239
240export const storageListDirectory = workspaceAction({212export const storageListDirectory = workspaceAction({
241 method: "POST",213 method: "POST",
242 args: {214 args: {
executor/packages/database/convex/http/mcp_auth.ts+4−39
@@ -38,29 +38,9 @@ function mcpAuthRequired(): boolean { · 37 unmodified lines
38 return isTruthyEnvValue(process.env.EXECUTOR_ENFORCE_MCP_AUTH);38 return isTruthyEnvValue(process.env.EXECUTOR_ENFORCE_MCP_AUTH);
39}39}
40 40
41function configuredMcpAudiences(): string[] {41// WorkOS AuthKit tokens use the requesting MCP client's client ID as the
42 const raw = (process.env.MCP_AUTH_AUDIENCE ?? "").trim();42// audience, which varies per client. Per WorkOS docs, only issuer + signature
43 if (!raw) {43// verification is required — audience checks are intentionally omitted.
44 return [];
45 }
46
47 return raw
48 .split(",")
49 .map((entry) => entry.trim())
50 .filter((entry) => entry.length > 0);
51}
52
53function deriveMcpAudiences(request: Request): string[] {
54 const url = new URL(request.url);
55 const derived = [
56 new URL(MCP_PATH, url.origin).toString(),
57 new URL(LEGACY_MCP_PATH, url.origin).toString(),
58 ];
59
60 const configured = configuredMcpAudiences();
61 const merged = [...configured, ...derived];
62 return Array.from(new Set(merged));
63}
64 44
65function parseWorkspaceId(raw: string): Id<"workspaces"> {45function parseWorkspaceId(raw: string): Id<"workspaces"> {
66 return raw as Id<"workspaces">;46 return raw as Id<"workspaces">;
@@ -165,7 +145,6 @@ export async function verifyMcpToken( · 98 unmodified lines
165 try {145 try {
166 const { payload } = await jwtVerify(token, config.jwks, {146 const { payload } = await jwtVerify(token, config.jwks, {
167 issuer: config.authorizationServer,147 issuer: config.authorizationServer,
168 audience: deriveMcpAudiences(request),
169 });148 });
170 if (typeof payload.sub === "string" && payload.sub.length > 0) {149 if (typeof payload.sub === "string" && payload.sub.length > 0) {
171 const providerClaim = typeof payload.provider === "string" ? payload.provider : undefined;150 const providerClaim = typeof payload.provider === "string" ? payload.provider : undefined;
@@ -177,21 +156,7 @@ export async function verifyMcpToken( · 5 unmodified lines
177 }156 }
178 }157 }
179 } catch (error) {158 } catch (error) {
180 // Decode the token payload (without verification) to log what we received vs expected159 console.error("MCP token verification failed:", error instanceof Error ? error.message : String(error));
181 try {
182 const [, payloadB64] = token.split(".");
183 if (payloadB64) {
184 const decoded = JSON.parse(atob(payloadB64.replace(/-/g, "+").replace(/_/g, "/")));
185 console.error("MCP token verification failed:", error instanceof Error ? error.message : String(error), {
186 tokenIssuer: decoded.iss,
187 tokenAudience: decoded.aud,
188 expectedIssuer: config.authorizationServer,
189 expectedAudiences: deriveMcpAudiences(request),
190 });
191 }
192 } catch {
193 console.error("MCP token verification failed:", error instanceof Error ? error.message : String(error));
194 }
195 }160 }
196 }161 }
197 162
executor/packages/database/src/runtime/tool_upgrade.ts+0−185
@@ -1,185 +0,0 @@
1import type { ActionCtx } from "../../convex/_generated/server";
2import { internal } from "../../convex/_generated/api";
3import type { Id } from "../../convex/_generated/dataModel.d.ts";
4import type { ToolSourceRecord } from "../../../core/src/types";
5import { normalizeExternalToolSource, loadSourceArtifact } from "./tool_source_loading";
6
7export interface OpenApiUpgradeDiffPreview {
8 sourceId: string;
9 currentSourceName: string;
10 proposedSourceName: string;
11 currentToolCount: number;
12 proposedToolCount: number;
13 addedCount: number;
14 removedCount: number;
15 changedCount: number;
16 hasChanges: boolean;
17 addedPaths: string[];
18 removedPaths: string[];
19 changedPaths: string[];
20 truncated: boolean;
21 warnings: string[];
22}
23
24const PREVIEW_LIST_LIMIT = 200;
25
26function clampPreview(items: string[]): { items: string[]; truncated: boolean } {
27 if (items.length <= PREVIEW_LIST_LIMIT) {
28 return { items, truncated: false };
29 }
30
31 return {
32 items: items.slice(0, PREVIEW_LIST_LIMIT),
33 truncated: true,
34 };
35}
36
37async function listCurrentSourceToolMap(
38 ctx: ActionCtx,
39 workspaceId: Id<"workspaces">,
40 sourceLabel: string,
41): Promise<Map<string, string>> {
42 const tools = new Map<string, string>();
43 let cursor: string | undefined;
44
45 while (true) {
46 const page: {
47 continueCursor: string | null;
48 items: Array<{ path: string }>;
49 } = await ctx.runQuery(internal.toolRegistry.listToolsBySourcePage, {
50 workspaceId,
51 source: sourceLabel,
52 cursor,
53 limit: 500,
54 });
55
56 const payloads: Array<{ path: string; serializedToolJson: string }> = await ctx.runQuery(
57 internal.toolRegistry.getSerializedToolsByPaths,
58 {
59 workspaceId,
60 paths: page.items.map((item) => item.path),
61 },
62 );
63
64 for (const payload of payloads) {
65 tools.set(payload.path, payload.serializedToolJson);
66 }
67
68 if (page.continueCursor === null) {
69 break;
70 }
71
72 cursor = page.continueCursor;
73 }
74
75 return tools;
76}
77
78export async function previewOpenApiSourceUpgradeForContext(
79 ctx: ActionCtx,
80 context: { workspaceId: Id<"workspaces">; accountId?: Id<"accounts">; clientId?: string },
81 args: { sourceId: string; name: string; config: Record<string, unknown> },
82): Promise<OpenApiUpgradeDiffPreview> {
83 const sources: ToolSourceRecord[] = await ctx.runQuery(internal.database.listToolSources, {
84 workspaceId: context.workspaceId,
85 });
86 const existing = sources.find((source) => source.id === args.sourceId);
87 if (!existing) {
88 throw new Error("Source not found");
89 }
90 if (existing.type !== "openapi") {
91 throw new Error("Upgrade preview is only supported for OpenAPI sources");
92 }
93
94 const candidateName = args.name.trim();
95 if (!candidateName) {
96 throw new Error("Source name is required");
97 }
98
99 const candidateSource: ToolSourceRecord = {
100 ...existing,
101 name: candidateName,
102 type: "openapi",
103 config: args.config,
104 enabled: true,
105 updatedAt: Date.now(),
106 };
107
108 const normalized = normalizeExternalToolSource(candidateSource);
109 if (normalized.isErr()) {
110 throw new Error(normalized.error.message);
111 }
112
113 if (normalized.value.type !== "openapi") {
114 throw new Error("Upgrade preview expects an OpenAPI source");
115 }
116
117 const loaded = await loadSourceArtifact(ctx, normalized.value, {
118 includeDts: false,
119 workspaceId: context.workspaceId,
120 accountId: context.accountId,
121 });
122
123 const proposedSerializedTools = loaded.artifact?.tools ?? [];
124 if (!loaded.artifact) {
125 const reason = loaded.warnings[0] ?? "No tools generated from proposed spec";
126 throw new Error(reason);
127 }
128
129 const proposedByPath = new Map<string, string>();
130 for (const tool of proposedSerializedTools) {
131 proposedByPath.set(tool.path, JSON.stringify(tool));
132 }
133
134 const currentByPath = await listCurrentSourceToolMap(
135 ctx,
136 context.workspaceId,
137 `openapi:${existing.name}`,
138 );
139
140 const added: string[] = [];
141 const removed: string[] = [];
142 const changed: string[] = [];
143
144 for (const [path, serialized] of proposedByPath.entries()) {
145 const current = currentByPath.get(path);
146 if (!current) {
147 added.push(path);
148 continue;
149 }
150 if (current !== serialized) {
151 changed.push(path);
152 }
153 }
154
155 for (const path of currentByPath.keys()) {
156 if (!proposedByPath.has(path)) {
157 removed.push(path);
158 }
159 }
160
161 added.sort((a, b) => a.localeCompare(b));
162 removed.sort((a, b) => a.localeCompare(b));
163 changed.sort((a, b) => a.localeCompare(b));
164
165 const cappedAdded = clampPreview(added);
166 const cappedRemoved = clampPreview(removed);
167 const cappedChanged = clampPreview(changed);
168
169 return {
170 sourceId: existing.id,
171 currentSourceName: existing.name,
172 proposedSourceName: candidateName,
173 currentToolCount: currentByPath.size,
174 proposedToolCount: proposedByPath.size,
175 addedCount: added.length,
176 removedCount: removed.length,
177 changedCount: changed.length,
178 hasChanges: added.length > 0 || removed.length > 0 || changed.length > 0,
179 addedPaths: cappedAdded.items,
180 removedPaths: cappedRemoved.items,
181 changedPaths: cappedChanged.items,
182 truncated: cappedAdded.truncated || cappedRemoved.truncated || cappedChanged.truncated,
183 warnings: loaded.warnings,
184 };
185}
executor/packages/runner-sandbox-host/src/index.ts+7−16
@@ -148,20 +148,9 @@ export class AgentFSStorageObject extends DurableObject<Env> { · 147 unmodified lines
148 throw new Error("sqlite.query rejects multi-statement SQL payloads");148 throw new Error("sqlite.query rejects multi-statement SQL payloads");
149 }149 }
150 150
151 if (mode === "read") {151 // Note: We rely on the application-level isReadOnlySql() check above
152 this.ctx.storage.sql.exec("PRAGMA query_only = 1");152 // instead of PRAGMA query_only, because Cloudflare Durable Object SQLite
153 try {153 // restricts PRAGMA usage and throws SQLITE_AUTH when setting query_only.
154 const cursor = this.ctx.storage.sql.exec(sql, ...params);
155 const rows = cursor.toArray().slice(0, Math.max(1, Math.floor(maxRows))) as Record<string, unknown>[];
156 return {
157 mode,
158 rows,
159 rowCount: rows.length,
160 };
161 } finally {
162 this.ctx.storage.sql.exec("PRAGMA query_only = 0");
163 }
164 }
165 154
166 const cursor = this.ctx.storage.sql.exec(sql, ...params);155 const cursor = this.ctx.storage.sql.exec(sql, ...params);
167 if (mode === "write") {156 if (mode === "write") {
@@ -172,10 +161,12 @@ export class AgentFSStorageObject extends DurableObject<Env> { · 4 unmodified lines
172 };161 };
173 }162 }
174 163
164 // read mode: return rows
165 const rows = cursor.toArray().slice(0, Math.max(1, Math.floor(maxRows))) as Record<string, unknown>[];
175 return {166 return {
176 mode,167 mode,
177 rows: [],168 rows,
178 rowCount: 0,169 rowCount: rows.length,
179 };170 };
180 }171 }
181 172