History

@executor/storage-postgres: relational storage for cloud

- Drizzle schema: users, teams, team_members, invitations, sources, tools, secrets, policies, plugin_kv - Implements ToolRegistryService, SecretStoreService, PolicyEngineService - AES-256-GCM encrypted secrets in DB - makePgKv for plugin compatibility via ScopedKv - User/team/invitation CRUD - 17 tests against PGlite - drizzle-orm in root catalog for single workspace copy

Rhys Sullivan committed 7 months ago

Commit 3e5de4a1 parent c2c5bb315 files · +136 −127Browse files

Unified · Split

15 files
  1. .gitignore +4 −1
  2. package.json +1 −0
  3. packages/core/storage-postgres/drizzle/0000_spicy_zemo.sql +94 −0
  4. packages/core/storage-postgres/drizzle/meta/0000_snapshot.json +1 −45
  5. packages/core/storage-postgres/drizzle/meta/_journal.json +2 −2
  6. packages/core/storage-postgres/package.json +1 −1
  7. packages/core/storage-postgres/src/index.test.ts +1 −21
  8. packages/core/storage-postgres/src/index.ts +4 −3
  9. packages/core/storage-postgres/src/pg-kv.ts +2 −2
  10. packages/core/storage-postgres/src/policy-engine.ts +2 −2
  11. packages/core/storage-postgres/src/schema.ts +0 −8
  12. packages/core/storage-postgres/src/secret-store.ts +11 −10
  13. packages/core/storage-postgres/src/tool-registry.ts +3 −3
  14. packages/core/storage-postgres/src/types.ts +7 −0
  15. packages/core/storage-postgres/src/user-store.ts +3 −29
.gitignore+4−1
@@ -39,4 +39,7 @@ executor.har · 38 unmodified lines
39.executor/39.executor/
40 40
41# desktop app build artifacts41# desktop app build artifacts
42apps/desktop/resources/42apps/desktop/resources/
43
44# cloud local dev database
45.pglite
package.json+1−0
@@ -41,6 +41,7 @@ · 40 unmodified lines
41 "@effect/vitest": "^0.27.0",41 "@effect/vitest": "^0.27.0",
42 "@types/node": "^24.3.1",42 "@types/node": "^24.3.1",
43 "bun-types": "^1.2.22",43 "bun-types": "^1.2.22",
44 "drizzle-orm": "^0.44.0",
44 "vitest": "^3.2.4"45 "vitest": "^3.2.4"
45 },46 },
46 "scripts": {47 "scripts": {
packages/core/storage-postgres/drizzle/0000_spicy_zemo.sql+94−0
@@ -0,0 +1,94 @@
1CREATE TABLE "invitations" (
2 "id" text PRIMARY KEY NOT NULL,
3 "team_id" text NOT NULL,
4 "email" text NOT NULL,
5 "invited_by" text NOT NULL,
6 "status" text NOT NULL,
7 "created_at" timestamp with time zone DEFAULT now() NOT NULL,
8 "expires_at" timestamp with time zone NOT NULL
9);
10--> statement-breakpoint
11CREATE TABLE "plugin_kv" (
12 "team_id" text NOT NULL,
13 "namespace" text NOT NULL,
14 "key" text NOT NULL,
15 "value" text NOT NULL,
16 CONSTRAINT "plugin_kv_team_id_namespace_key_pk" PRIMARY KEY("team_id","namespace","key")
17);
18--> statement-breakpoint
19CREATE TABLE "policies" (
20 "id" text NOT NULL,
21 "team_id" text NOT NULL,
22 "name" text NOT NULL,
23 "action" text NOT NULL,
24 "match_tool_pattern" text,
25 "match_source_id" text,
26 "priority" integer NOT NULL,
27 "created_at" timestamp with time zone DEFAULT now() NOT NULL,
28 CONSTRAINT "policies_id_team_id_pk" PRIMARY KEY("id","team_id")
29);
30--> statement-breakpoint
31CREATE TABLE "secrets" (
32 "id" text NOT NULL,
33 "team_id" text NOT NULL,
34 "name" text NOT NULL,
35 "purpose" text,
36 "encrypted_value" "bytea" NOT NULL,
37 "iv" "bytea" NOT NULL,
38 "created_at" timestamp with time zone DEFAULT now() NOT NULL,
39 CONSTRAINT "secrets_id_team_id_pk" PRIMARY KEY("id","team_id")
40);
41--> statement-breakpoint
42CREATE TABLE "sources" (
43 "id" text NOT NULL,
44 "team_id" text NOT NULL,
45 "name" text NOT NULL,
46 "kind" text NOT NULL,
47 "config" jsonb NOT NULL,
48 "created_at" timestamp with time zone DEFAULT now() NOT NULL,
49 CONSTRAINT "sources_id_team_id_pk" PRIMARY KEY("id","team_id")
50);
51--> statement-breakpoint
52CREATE TABLE "team_members" (
53 "team_id" text NOT NULL,
54 "user_id" text NOT NULL,
55 "role" text NOT NULL,
56 "created_at" timestamp with time zone DEFAULT now() NOT NULL,
57 CONSTRAINT "team_members_team_id_user_id_pk" PRIMARY KEY("team_id","user_id")
58);
59--> statement-breakpoint
60CREATE TABLE "teams" (
61 "id" text PRIMARY KEY NOT NULL,
62 "name" text NOT NULL,
63 "created_at" timestamp with time zone DEFAULT now() NOT NULL
64);
65--> statement-breakpoint
66CREATE TABLE "tool_definitions" (
67 "name" text NOT NULL,
68 "team_id" text NOT NULL,
69 "schema" jsonb NOT NULL,
70 CONSTRAINT "tool_definitions_name_team_id_pk" PRIMARY KEY("name","team_id")
71);
72--> statement-breakpoint
73CREATE TABLE "tools" (
74 "id" text NOT NULL,
75 "team_id" text NOT NULL,
76 "source_id" text NOT NULL,
77 "plugin_key" text NOT NULL,
78 "name" text NOT NULL,
79 "description" text,
80 "may_elicit" boolean,
81 "input_schema" jsonb,
82 "output_schema" jsonb,
83 "created_at" timestamp with time zone DEFAULT now() NOT NULL,
84 CONSTRAINT "tools_id_team_id_pk" PRIMARY KEY("id","team_id")
85);
86--> statement-breakpoint
87CREATE TABLE "users" (
88 "id" text PRIMARY KEY NOT NULL,
89 "email" text NOT NULL,
90 "name" text,
91 "avatar_url" text,
92 "created_at" timestamp with time zone DEFAULT now() NOT NULL,
93 CONSTRAINT "users_email_unique" UNIQUE("email")
94);
packages/core/storage-postgres/drizzle/meta/0000_snapshot.json+1−45
@@ -1,5 +1,5 @@
1{1{
2 "id": "36e0d08d-3eb6-467a-8440-e39b7bfee61e",2 "id": "ee7b4779-1928-46a3-bcb8-1a48898bec19",
3 "prevId": "00000000-0000-0000-0000-000000000000",3 "prevId": "00000000-0000-0000-0000-000000000000",
4 "version": "7",4 "version": "7",
5 "dialect": "postgresql",5 "dialect": "postgresql",
@@ -240,50 +240,6 @@ · 234 unmodified lines
240 "checkConstraints": {},240 "checkConstraints": {},
241 "isRLSEnabled": false241 "isRLSEnabled": false
242 },242 },
243 "public.sessions": {
244 "name": "sessions",
245 "schema": "",
246 "columns": {
247 "id": {
248 "name": "id",
249 "type": "text",
250 "primaryKey": true,
251 "notNull": true
252 },
253 "user_id": {
254 "name": "user_id",
255 "type": "text",
256 "primaryKey": false,
257 "notNull": true
258 },
259 "team_id": {
260 "name": "team_id",
261 "type": "text",
262 "primaryKey": false,
263 "notNull": true
264 },
265 "expires_at": {
266 "name": "expires_at",
267 "type": "timestamp with time zone",
268 "primaryKey": false,
269 "notNull": true
270 },
271 "created_at": {
272 "name": "created_at",
273 "type": "timestamp with time zone",
274 "primaryKey": false,
275 "notNull": true,
276 "default": "now()"
277 }
278 },
279 "indexes": {},
280 "foreignKeys": {},
281 "compositePrimaryKeys": {},
282 "uniqueConstraints": {},
283 "policies": {},
284 "checkConstraints": {},
285 "isRLSEnabled": false
286 },
287 "public.sources": {243 "public.sources": {
288 "name": "sources",244 "name": "sources",
289 "schema": "",245 "schema": "",
packages/core/storage-postgres/drizzle/meta/_journal.json+2−2
@@ -5,8 +5,8 @@ · 4 unmodified lines
5 {5 {
6 "idx": 0,6 "idx": 0,
7 "version": "7",7 "version": "7",
8 "when": 1775542409153,8 "when": 1775552333040,
9 "tag": "0000_great_speedball",9 "tag": "0000_spicy_zemo",
10 "breakpoints": true10 "breakpoints": true
11 }11 }
12 ]12 ]
packages/core/storage-postgres/package.json+1−1
@@ -10,7 +10,7 @@ · 9 unmodified lines
10 "@effect/sql": "catalog:",10 "@effect/sql": "catalog:",
11 "@effect/sql-pg": "^0.28.0",11 "@effect/sql-pg": "^0.28.0",
12 "@executor/sdk": "workspace:*",12 "@executor/sdk": "workspace:*",
13 "drizzle-orm": "^0.45.2",13 "drizzle-orm": "catalog:",
14 "effect": "catalog:",14 "effect": "catalog:",
15 "pg": "^8.16.0"15 "pg": "^8.16.0"
16 },16 },
packages/core/storage-postgres/src/index.test.ts+1−21
@@ -46,7 +46,7 @@ beforeAll(async () => { · 45 unmodified lines
46 46
47beforeEach(async () => {47beforeEach(async () => {
48 await db.execute(48 await db.execute(
49 sql`TRUNCATE plugin_kv, policies, secrets, tool_definitions, tools, sources, sessions, invitations, team_members, teams, users`,49 sql`TRUNCATE plugin_kv, policies, secrets, tool_definitions, tools, sources, invitations, team_members, teams, users`,
50 );50 );
51});51});
52 52
@@ -408,24 +408,4 @@ describe("UserStore", () => { · 355 unmodified lines
408 expect(await store.getPendingInvitations("new@example.com")).toHaveLength(0);408 expect(await store.getPendingInvitations("new@example.com")).toHaveLength(0);
409 });409 });
410 410
411 it("session management", async () => {
412 const store = makeUserStore(db);
413 await store.upsertUser({ id: "u1", email: "a@example.com" });
414 const team = await store.createTeam("Team");
415
416 const session = await store.createSession("u1", team.id, new Date(Date.now() + 3600_000));
417 expect((await store.getSession(session.id))!.userId).toBe("u1");
418
419 await store.deleteSession(session.id);
420 expect(await store.getSession(session.id)).toBeNull();
421 });
422
423 it("expired sessions return null", async () => {
424 const store = makeUserStore(db);
425 await store.upsertUser({ id: "u1", email: "a@example.com" });
426 const team = await store.createTeam("Team");
427
428 const session = await store.createSession("u1", team.id, new Date(Date.now() - 1000));
429 expect(await store.getSession(session.id)).toBeNull();
430 });
431});411});
packages/core/storage-postgres/src/index.ts+4−3
@@ -17,9 +17,9 @@ · 16 unmodified lines
17//17//
18// ---------------------------------------------------------------------------18// ---------------------------------------------------------------------------
19 19
20import type { PgDatabase } from "drizzle-orm/pg-core";
21import { ScopeId, makeInMemorySourceRegistry } from "@executor/sdk";20import { ScopeId, makeInMemorySourceRegistry } from "@executor/sdk";
22import type { Scope, ExecutorConfig, ExecutorPlugin } from "@executor/sdk";21import type { Scope, ExecutorConfig, ExecutorPlugin } from "@executor/sdk";
22import type { DrizzleDb } from "./types";
23 23
24import { makePgToolRegistry } from "./tool-registry";24import { makePgToolRegistry } from "./tool-registry";
25import { makePgSecretStore } from "./secret-store";25import { makePgSecretStore } from "./secret-store";
@@ -30,8 +30,9 @@ export { makePgToolRegistry } from "./tool-registry"; · 4 unmodified lines
30export { makePgSecretStore } from "./secret-store";30export { makePgSecretStore } from "./secret-store";
31export { makePgPolicyEngine } from "./policy-engine";31export { makePgPolicyEngine } from "./policy-engine";
32export { makeUserStore } from "./user-store";32export { makeUserStore } from "./user-store";
33export type { User, Team, TeamMember, Invitation, Session } from "./user-store";33export type { User, Team, TeamMember, Invitation } from "./user-store";
34export { encrypt, decrypt } from "./crypto";34export { encrypt, decrypt } from "./crypto";
35export type { DrizzleDb } from "./types";
35export * from "./schema";36export * from "./schema";
36 37
37// ---------------------------------------------------------------------------38// ---------------------------------------------------------------------------
@@ -41,7 +42,7 @@ export * from "./schema"; · 3 unmodified lines
41export const makePgConfig = <42export const makePgConfig = <
42 const TPlugins extends readonly ExecutorPlugin<string, object>[] = [],43 const TPlugins extends readonly ExecutorPlugin<string, object>[] = [],
43>(44>(
44 db: PgDatabase<any, any, any>,45 db: DrizzleDb,
45 options: {46 options: {
46 readonly teamId: string;47 readonly teamId: string;
47 readonly teamName: string;48 readonly teamName: string;
packages/core/storage-postgres/src/pg-kv.ts+2−2
@@ -4,12 +4,12 @@ · 3 unmodified lines
4 4
5import { Effect } from "effect";5import { Effect } from "effect";
6import { eq, and } from "drizzle-orm";6import { eq, and } from "drizzle-orm";
7import type { PgDatabase } from "drizzle-orm/pg-core";
8import type { Kv } from "@executor/sdk";7import type { Kv } from "@executor/sdk";
9 8
10import { pluginKv } from "./schema";9import { pluginKv } from "./schema";
10import type { DrizzleDb } from "./types";
11 11
12export const makePgKv = (db: PgDatabase<any, any, any>, teamId: string): Kv => ({12export const makePgKv = (db: DrizzleDb, teamId: string): Kv => ({
13 get: (namespace, key) =>13 get: (namespace, key) =>
14 Effect.tryPromise(async () => {14 Effect.tryPromise(async () => {
15 const rows = await db15 const rows = await db
packages/core/storage-postgres/src/policy-engine.ts+2−2
@@ -4,15 +4,15 @@ · 3 unmodified lines
4 4
5import { Effect } from "effect";5import { Effect } from "effect";
6import { eq, and } from "drizzle-orm";6import { eq, and } from "drizzle-orm";
7import type { PgDatabase } from "drizzle-orm/pg-core";
8 7
9import { Policy, PolicyId, ScopeId } from "@executor/sdk";8import { Policy, PolicyId, ScopeId } from "@executor/sdk";
9import type { DrizzleDb } from "./types";
10import type { PolicyCheckInput } from "@executor/sdk";10import type { PolicyCheckInput } from "@executor/sdk";
11 11
12import { policies } from "./schema";12import { policies } from "./schema";
13 13
14export const makePgPolicyEngine = (14export const makePgPolicyEngine = (
15 db: PgDatabase<any, any, any>,15 db: DrizzleDb,
16 teamId: string,16 teamId: string,
17) => {17) => {
18 let counter = 0;18 let counter = 0;
packages/core/storage-postgres/src/schema.ts+0−8
@@ -58,14 +58,6 @@ export const invitations = pgTable("invitations", { · 57 unmodified lines
58 expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(),58 expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(),
59});59});
60 60
61export const sessions = pgTable("sessions", {
62 id: text("id").primaryKey().$defaultFn(() => crypto.randomUUID()),
63 userId: text("user_id").notNull(),
64 teamId: text("team_id").notNull(),
65 expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(),
66 createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
67});
68
69// ---------------------------------------------------------------------------61// ---------------------------------------------------------------------------
70// Domain data — all team-scoped62// Domain data — all team-scoped
71// ---------------------------------------------------------------------------63// ---------------------------------------------------------------------------
packages/core/storage-postgres/src/secret-store.ts+11−10
@@ -4,9 +4,9 @@ · 3 unmodified lines
4 4
5import { Effect, Option } from "effect";5import { Effect, Option } from "effect";
6import { eq, and } from "drizzle-orm";6import { eq, and } from "drizzle-orm";
7import type { PgDatabase } from "drizzle-orm/pg-core";
8 7
9import { SecretRef, SecretId, ScopeId } from "@executor/sdk";8import { SecretRef, SecretId, ScopeId } from "@executor/sdk";
9import type { DrizzleDb } from "./types";
10import { SecretNotFoundError, SecretResolutionError } from "@executor/sdk";10import { SecretNotFoundError, SecretResolutionError } from "@executor/sdk";
11import type { SecretProvider, SetSecretInput } from "@executor/sdk";11import type { SecretProvider, SetSecretInput } from "@executor/sdk";
12 12
@@ -14,7 +14,7 @@ import { secrets } from "./schema"; · 1 unmodified line
14import { encrypt, decrypt } from "./crypto";14import { encrypt, decrypt } from "./crypto";
15 15
16export const makePgSecretStore = (16export const makePgSecretStore = (
17 db: PgDatabase<any, any, any>,17 db: DrizzleDb,
18 teamId: string,18 teamId: string,
19 encryptionKey: string,19 encryptionKey: string,
20) => {20) => {
@@ -99,14 +99,15 @@ export const makePgSecretStore = ( · 78 unmodified lines
99 99
100 const row = rows[0];100 const row = rows[0];
101 if (row) {101 if (row) {
102 try {102 const decrypted = yield* Effect.try({
103 return decrypt(row.encryptedValue, row.iv, encryptionKey);103 try: () => decrypt(row.encryptedValue, row.iv, encryptionKey),
104 } catch {104 catch: () =>
105 return yield* new SecretResolutionError({105 new SecretResolutionError({
106 secretId,106 secretId,
107 message: `Failed to decrypt secret "${secretId}"`,107 message: `Failed to decrypt secret "${secretId}"`,
108 });108 }),
109 }109 });
110 return decrypted;
110 }111 }
111 112
112 // Fall back to registered providers113 // Fall back to registered providers
packages/core/storage-postgres/src/tool-registry.ts+3−3
@@ -3,10 +3,10 @@ · 2 unmodified lines
3// ---------------------------------------------------------------------------3// ---------------------------------------------------------------------------
4 4
5import { Effect } from "effect";5import { Effect } from "effect";
6import { eq, and, or, ilike } from "drizzle-orm";6import { eq, and } from "drizzle-orm";
7import type { PgDatabase } from "drizzle-orm/pg-core";
8 7
9import type { ToolId } from "@executor/sdk";8import type { ToolId } from "@executor/sdk";
9import type { DrizzleDb } from "./types";
10import { ToolNotFoundError, ToolInvocationError, ToolRegistration } from "@executor/sdk";10import { ToolNotFoundError, ToolInvocationError, ToolRegistration } from "@executor/sdk";
11import { buildToolTypeScriptPreview } from "@executor/sdk";11import { buildToolTypeScriptPreview } from "@executor/sdk";
12import type {12import type {
@@ -19,7 +19,7 @@ import type { · 6 unmodified lines
19import { tools, toolDefinitions } from "./schema";19import { tools, toolDefinitions } from "./schema";
20 20
21export const makePgToolRegistry = (21export const makePgToolRegistry = (
22 db: PgDatabase<any, any, any>,22 db: DrizzleDb,
23 teamId: string,23 teamId: string,
24) => {24) => {
25 const runtimeTools = new Map<string, ToolRegistration>();25 const runtimeTools = new Map<string, ToolRegistration>();
packages/core/storage-postgres/src/types.ts+7−0
@@ -0,0 +1,7 @@
1// ---------------------------------------------------------------------------
2// Shared Drizzle DB type
3// ---------------------------------------------------------------------------
4
5import type { PgDatabase } from "drizzle-orm/pg-core";
6
7export type DrizzleDb = PgDatabase<any, any, any>;
packages/core/storage-postgres/src/user-store.ts+3−29
@@ -3,17 +3,16 @@ · 2 unmodified lines
3// ---------------------------------------------------------------------------3// ---------------------------------------------------------------------------
4 4
5import { eq, and } from "drizzle-orm";5import { eq, and } from "drizzle-orm";
6import type { PgDatabase } from "drizzle-orm/pg-core";
7 6
8import { users, teams, teamMembers, invitations, sessions } from "./schema";7import { users, teams, teamMembers, invitations } from "./schema";
8import type { DrizzleDb } from "./types";
9 9
10export type User = typeof users.$inferSelect;10export type User = typeof users.$inferSelect;
11export type Team = typeof teams.$inferSelect;11export type Team = typeof teams.$inferSelect;
12export type TeamMember = typeof teamMembers.$inferSelect;12export type TeamMember = typeof teamMembers.$inferSelect;
13export type Invitation = typeof invitations.$inferSelect;13export type Invitation = typeof invitations.$inferSelect;
14export type Session = typeof sessions.$inferSelect;
15 14
16export const makeUserStore = (db: PgDatabase<any, any, any>) => ({15export const makeUserStore = (db: DrizzleDb) => ({
17 // --- Users ---16 // --- Users ---
18 17
19 upsertUser: async (user: { id: string; email: string; name?: string; avatarUrl?: string }) => {18 upsertUser: async (user: { id: string; email: string; name?: string; avatarUrl?: string }) => {
@@ -133,29 +132,4 @@ export const makeUserStore = (db: PgDatabase<any, any, any>) => ({ · 113 unmodified lines
133 return updated ?? null;132 return updated ?? null;
134 },133 },
135 134
136 // --- Sessions ---
137
138 createSession: async (userId: string, teamId: string, expiresAt: Date) => {
139 const [session] = await db
140 .insert(sessions)
141 .values({ userId, teamId, expiresAt })
142 .returning();
143 return session!;
144 },
145
146 getSession: async (sessionId: string) => {
147 const rows = await db.select().from(sessions).where(eq(sessions.id, sessionId));
148 const session = rows[0];
149 if (!session) return null;
150 if (session.expiresAt < new Date()) return null;
151 return session;
152 },
153
154 deleteSession: async (sessionId: string) => {
155 await db.delete(sessions).where(eq(sessions.id, sessionId));
156 },
157
158 deleteUserSessions: async (userId: string) => {
159 await db.delete(sessions).where(eq(sessions.userId, userId));
160 },
161});135});