History

Sync tool inventory via worker watchers

Track workspace tool-source updates in Convex and have the worker refresh cached tool inventory so the web UI stays live without calling /api/tools for warmup.

Rhys Sullivan committed 9 months ago

Commit 9783c8d1 parent 1ced8666 files · +77 −29Browse files

Unified · Split

6 files
  1. executor/apps/server/src/database.ts +5 −0
  2. executor/apps/server/src/service.ts +8 −0
  3. executor/apps/server/src/worker.ts +41 −0
  4. executor/apps/web/src/components/tools-view.tsx +1 −12
  5. executor/apps/web/src/hooks/use-workspace-tools.ts +3 −17
  6. executor/convex/database.ts +19 −0
executor/apps/server/src/database.ts+5−0
@@ -45,6 +45,7 @@ type QueryName = · 44 unmodified lines
4545 | "database:listCredentials"
4646 | "database:resolveCredential"
4747 | "database:listToolSources"
48+ | "database:listToolSourceWorkspaceUpdates"
4849 | "database:listWorkspaceToolsForContext"
4950 | "database:listTaskEvents";
5051
@@ -210,6 +211,10 @@ export class ExecutorDatabase { · 159 unmodified lines
210211 return await this.query("database:listToolSources", { workspaceId });
211212 }
212213
214+ async listToolSourceWorkspaceUpdates(): Promise<Array<{ workspaceId: string; updatedAt: number }>> {
215+ return await this.query("database:listToolSourceWorkspaceUpdates", {});
216+ }
217+
213218 async syncWorkspaceTools(params: {
214219 workspaceId: string;
215220 tools: ToolDescriptor[];
executor/apps/server/src/service.ts+8−0
@@ -213,6 +213,14 @@ export class ExecutorService { · 212 unmodified lines
213213 return await this.db.listToolSources(workspaceId);
214214 }
215215
216+ async listToolSourceWorkspaceUpdates(): Promise<Array<{ workspaceId: string; updatedAt: number }>> {
217+ return await this.db.listToolSourceWorkspaceUpdates();
218+ }
219+
220+ async refreshWorkspaceTools(workspaceId: string): Promise<void> {
221+ await this.getWorkspaceTools(workspaceId);
222+ }
223+
216224 async upsertToolSource(input: {
217225 id?: string;
218226 workspaceId: string;
executor/apps/server/src/worker.ts+41−0
@@ -51,6 +51,7 @@ const convexClient = new ConvexClient(convexUrl, { · 50 unmodified lines
5151 });
5252
5353 let draining = false;
54+let syncingToolInventories = false;
5455
5556 async function drainQueue(trigger: string): Promise<void> {
5657 if (draining) {
@@ -78,6 +79,30 @@ async function drainQueue(trigger: string): Promise<void> { · 21 unmodified lines
7879 }
7980 }
8081
82+async function syncWorkspaceToolInventories(trigger: string): Promise<void> {
83+ if (syncingToolInventories) {
84+ return;
85+ }
86+
87+ syncingToolInventories = true;
88+ try {
89+ const updates = await service.listToolSourceWorkspaceUpdates();
90+ if (updates.length === 0) {
91+ return;
92+ }
93+
94+ console.log(`[worker] ${trigger}: syncing tool inventories for ${updates.length} workspace(s)`);
95+ for (const update of updates) {
96+ await service.refreshWorkspaceTools(update.workspaceId);
97+ }
98+ } catch (error) {
99+ const message = error instanceof Error ? error.message : String(error);
100+ console.error(`[worker] tool inventory sync failed: ${message}`);
101+ } finally {
102+ syncingToolInventories = false;
103+ }
104+}
105+
81106 const queueSubscription = convexClient.onUpdate(
82107 api.database.listQueuedTaskIds,
83108 { limit: 1 },
@@ -91,15 +116,31 @@ const queueSubscription = convexClient.onUpdate( · 7 unmodified lines
91116 },
92117 );
93118
119+const toolSourceSubscription = convexClient.onUpdate(
120+ api.database.listToolSourceWorkspaceUpdates,
121+ {},
122+ (updates) => {
123+ if (updates.length > 0) {
124+ void syncWorkspaceToolInventories("onUpdate");
125+ }
126+ },
127+ (error) => {
128+ console.warn(`[worker] tool source watcher error: ${error.message}`);
129+ },
130+);
131+
94132 const interval = setInterval(() => {
95133 void drainQueue("interval");
134+ void syncWorkspaceToolInventories("interval");
96135 }, pollMs);
97136
98137 await drainQueue("startup");
138+await syncWorkspaceToolInventories("startup");
99139
100140 function shutdown(signal: string): void {
101141 console.log(`[worker] received ${signal}, shutting down...`);
102142 queueSubscription.unsubscribe();
143+ toolSourceSubscription.unsubscribe();
103144 clearInterval(interval);
104145 void convexClient.close();
105146 process.exit(0);
executor/apps/web/src/components/tools-view.tsx+1−12
@@ -5,7 +5,6 @@ import { · 4 unmodified lines
55 Wrench,
66 Plus,
77 Trash2,
8− RefreshCw,
98 ShieldCheck,
109 Globe,
1110 Server,
@@ -699,17 +698,7 @@ export function ToolsView() { · 687 unmodified lines
699698 <PageHeader
700699 title="Tools"
701700 description="Manage tool sources and view available tools"
702− >
703− <Button
704− variant="outline"
705− size="sm"
706− className="h-8 text-xs"
707− onClick={() => void refreshInventory()}
708− >
709− <RefreshCw className="h-3.5 w-3.5 mr-1.5" />
710− Refresh
711− </Button>
712− </PageHeader>
701+ />
713702
714703 <Tabs defaultValue="sources" className="w-full">
715704 <TabsList className="bg-muted/50 h-9">
executor/apps/web/src/hooks/use-workspace-tools.ts+3−17
@@ -1,9 +1,8 @@
11 "use client";
22
3−import { useCallback, useEffect } from "react";
3+import { useCallback } from "react";
44 import { useQuery } from "convex/react";
55 import { convexApi } from "../lib/convex-api";
6−import * as api from "../lib/api";
76
87 interface WorkspaceContext {
98 workspaceId: string;
@@ -24,21 +23,8 @@ export function useWorkspaceTools(context: WorkspaceContext | null) { · 14 unmodified lines
2423 );
2524
2625 const refresh = useCallback(async () => {
27− if (!context) return;
28− try {
29− await api.listToolsForContext({
30− workspaceId: context.workspaceId,
31− actorId: context.actorId,
32− clientId: context.clientId,
33− });
34− } catch {
35− // best effort warm-up for server-side tool discovery sync
36− }
37− }, [context]);
38−
39− useEffect(() => {
40− void refresh();
41− }, [refresh]);
26+ // Convex queries are live; manual refresh is not required.
27+ }, []);
4228
4329 return {
4430 tools,
executor/convex/database.ts+19−0
@@ -763,6 +763,25 @@ export const listToolSources = query({ · 762 unmodified lines
763763 },
764764 });
765765
766+export const listToolSourceWorkspaceUpdates = query({
767+ args: {},
768+ handler: async (ctx) => {
769+ const docs = await ctx.db.query("toolSources").collect();
770+ const byWorkspace = new Map<string, number>();
771+
772+ for (const doc of docs) {
773+ const existing = byWorkspace.get(doc.workspaceId) ?? 0;
774+ if (doc.updatedAt > existing) {
775+ byWorkspace.set(doc.workspaceId, doc.updatedAt);
776+ }
777+ }
778+
779+ return [...byWorkspace.entries()]
780+ .map(([workspaceId, updatedAt]) => ({ workspaceId, updatedAt }))
781+ .sort((a, b) => a.workspaceId.localeCompare(b.workspaceId));
782+ },
783+});
784+
766785 export const syncWorkspaceTools = mutation({
767786 args: {
768787 workspaceId: v.string(),