feat(toolkits): per-toolkit policies + access-mode enforcement (slice 4)

Add a toolkit_policies collection (pattern -> approve|require_approval|block) to the plugin + CRUD/view. resolveScope returns the policies; the Executor wrapper enforces `block` as exclusion (the tool is removed from the slice and blocked at execute). Read-only access filters to tools annotated readOnly (fail-closed: unclassified counts as write). approve/require_approval are persisted + surfaced; org-level approval policies still apply via the base executor (toolkit-level approval enforcement is a follow-up). e2e (selfhost): full runs the tool; a block policy stops it; read-only access blocks an unclassified tool.

Rhys Sullivan committed 4 months ago

Commit d059ee81 parent 2f9241d5 files · +240 −21Browse files

Unified · Split

5 files

e2e/selfhost/toolkits-policies.test.ts +121 −0
packages/core/sdk/src/index.ts +2 −0
packages/core/sdk/src/toolkit-scope.ts +39 −1
packages/plugins/toolkits/src/server.ts +63 −20
packages/plugins/toolkits/src/shared.ts +15 −0

e2e/selfhost/toolkits-policies.test.ts+121−0

@@ -0,0 +1,121 @@
	1	+// Selfhost · Slice 4: per-toolkit policies + access modes at execute time.
	2	+// - full access -> the connection's tool runs;
	3	+// - a `block` policy on that tool -> blocked at execute (exclusion);
	4	+// - read-only access -> an unclassified (non-read-only) tool is blocked
	5	+// (fail-closed: unclassified counts as write).
	6	+// One greeting connection, three toolkits over it, compared via MCP execute.
	7	+import { randomBytes } from "node:crypto";
	8	+
	9	+import { expect } from "@effect/vitest";
	10	+import { Effect } from "effect";
	11	+import { composePluginApi } from "@executor-js/api/server";
	12	+import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api";
	13	+import { makeGreetingMcpServer, serveMcpServer } from "@executor-js/plugin-mcp/testing";
	14	+import { toolkitsPlugin } from "@executor-js/plugin-toolkits/server";
	15	+import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
	16	+
	17	+import { scenario } from "../src/scenario";
	18	+import { Api, Mcp, Target } from "../src/services";
	19	+
	20	+const api = composePluginApi([mcpHttpPlugin(), toolkitsPlugin()] as const);
	21	+const ident = (prefix: string): string => `${prefix}${randomBytes(4).toString("hex")}`;
	22	+
	23	+scenario(
	24	+ "Toolkits · access modes + block policies are enforced at execute",
	25	+ { timeout: 180_000 },
	26	+ Effect.scoped(
	27	+ Effect.gen(function* () {
	28	+ const target = yield* Target;
	29	+ const { client: makeApiClient } = yield* Api;
	30	+ const mcp = yield* Mcp;
	31	+ const identity = yield* target.newIdentity();
	32	+ const client = yield* makeApiClient(api, identity);
	33	+
	34	+ const slug = ident("pol");
	35	+ const conn = ident("conn");
	36	+ const token = `tok-${randomBytes(6).toString("hex")}`;
	37	+ const server = yield* serveMcpServer(() => makeGreetingMcpServer(), {
	38	+ auth: {
	39	+ validateAuthorization: (authorization) =>
	40	+ Effect.succeed(authorization === `Bearer ${token}`),
	41	+ },
	42	+ });
	43	+ yield* client.mcp.addServer({
	44	+ payload: {
	45	+ transport: "remote",
	46	+ name: `Greeting ${slug}`,
	47	+ endpoint: server.endpoint,
	48	+ slug,
	49	+ authenticationTemplate: [
	50	+ {
	51	+ type: "apiKey",
	52	+ headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] },
	53	+ },
	54	+ ],
	55	+ },
	56	+ });
	57	+ yield* client.connections.create({
	58	+ payload: {
	59	+ owner: "org",
	60	+ name: ConnectionName.make(conn),
	61	+ integration: IntegrationSlug.make(slug),
	62	+ template: AuthTemplateSlug.make("header"),
	63	+ value: token,
	64	+ },
	65	+ });
	66	+
	67	+ const full = yield* client.toolkits.create({
	68	+ payload: {
	69	+ slug: ident("kitfull"),
	70	+ name: "Full",
	71	+ scope: "workspace",
	72	+ connections: [
	73	+ { integration: IntegrationSlug.make(slug), connection: conn, access: "full" },
	74	+ ],
	75	+ },
	76	+ });
	77	+ const blocked = yield* client.toolkits.create({
	78	+ payload: {
	79	+ slug: ident("kitblock"),
	80	+ name: "Blocked",
	81	+ scope: "workspace",
	82	+ connections: [
	83	+ { integration: IntegrationSlug.make(slug), connection: conn, access: "full" },
	84	+ ],
	85	+ policies: [{ pattern: `${slug}.${conn}.simple_echo`, action: "block" }],
	86	+ },
	87	+ });
	88	+ const readonly = yield* client.toolkits.create({
	89	+ payload: {
	90	+ slug: ident("kitread"),
	91	+ name: "Read",
	92	+ scope: "workspace",
	93	+ connections: [
	94	+ { integration: IntegrationSlug.make(slug), connection: conn, access: "read" },
	95	+ ],
	96	+ },
	97	+ });
	98	+
	99	+ const code = `return await tools.${slug}.org.${conn}.simple_echo({});`;
	100	+ const fullCall = yield* mcp
	101	+ .session(identity, { toolkit: full.slug })
	102	+ .call("execute", { code });
	103	+ const blockCall = yield* mcp
	104	+ .session(identity, { toolkit: blocked.slug })
	105	+ .call("execute", { code });
	106	+ const readCall = yield* mcp
	107	+ .session(identity, { toolkit: readonly.slug })
	108	+ .call("execute", { code });
	109	+
	110	+ expect(fullCall.ok, `full access runs the tool; text=${fullCall.text}`).toBe(true);
	111	+ expect(
	112	+ blockCall.text,
	113	+ `block policy must stop the tool; full=${fullCall.text} block=${blockCall.text}`,
	114	+ ).not.toBe(fullCall.text);
	115	+ expect(
	116	+ readCall.text,
	117	+ `read-only must hide an unclassified (write) tool; full=${fullCall.text} read=${readCall.text}`,
	118	+ ).not.toBe(fullCall.text);
	119	+ }),
	120	+ ),
	121	+);

packages/core/sdk/src/index.ts+2−0

@@ -358,6 +358,8 @@ export { · 357 unmodified lines
358	358	EMPTY_TOOLKIT_SCOPE,
359	359	type ResolvedToolkitScope,
360	360	type ToolkitAccess,
	361	+ type ToolkitPolicyAction,
	362	+ type ToolkitPolicyRule,
361	363	type ToolkitResolver,
362	364	type ToolkitScopeEntry,
363	365	} from "./toolkit-scope";

packages/core/sdk/src/toolkit-scope.ts+39−1

@@ -19,6 +19,15 @@ import type { StorageFailure } from "./fuma-runtime"; · 18 unmodified lines
19	19
20	20	export type ToolkitAccess = "off" \| "read" \| "full";
21	21
	22	+export type ToolkitPolicyAction = "approve" \| "require_approval" \| "block";
	23	+
	24	+export interface ToolkitPolicyRule {
	25	+ /** Glob over `<integration>.<connection>.<tool>` — `*` matches one segment,
	26	+ * a trailing `` matches the rest, and `` inside a segment globs. */
	27	+ readonly pattern: string;
	28	+ readonly action: ToolkitPolicyAction;
	29	+}
	30	+
22	31	export interface ToolkitScopeEntry {
23	32	readonly integration: string;
24	33	/** A pinned connection name, or "" to track every connection of the integration. /
@@ -28,13 +37,37 @@ export interface ToolkitScopeEntry { · 3 unmodified lines
28	37
29	38	export interface ResolvedToolkitScope {
30	39	readonly entries: readonly ToolkitScopeEntry[];
	40	+ /** Per-toolkit policy rules. v1 enforces `block` (exclusion); `approve` /
	41	+ * `require_approval` are carried for the briefing — org-level approval
	42	+ * policies are still enforced by the base executor when inherited. */
	43	+ readonly policies: readonly ToolkitPolicyRule[];
31	44	readonly inheritOrgPolicies: boolean;
32	45	}
33	46
34	47	/** Empty slice — exposes no connection tools (static/core tools stay visible).
35	48	* Applied fail-closed when a selector resolves to no toolkit (unknown or
36	49	* not visible to the caller), so a bad selector never leaks the full account. */
37		−export const EMPTY_TOOLKIT_SCOPE: ResolvedToolkitScope = { entries: [], inheritOrgPolicies: true };
	50	+export const EMPTY_TOOLKIT_SCOPE: ResolvedToolkitScope = {
	51	+ entries: [],
	52	+ policies: [],
	53	+ inheritOrgPolicies: true,
	54	+};
	55	+
	56	+/** Glob match over dot segments: `` matches one segment, a trailing bare ``
	57	+ * matches the rest, and `` inside a segment globs within it. /
	58	+const matchPattern = (pattern: string, target: string): boolean => {
	59	+ const p = pattern.toLowerCase().trim().split(".");
	60	+ const a = target.toLowerCase().split(".");
	61	+ for (let i = 0; i < p.length; i++) {
	62	+ if (p[i] === "*" && i === p.length - 1) return true;
	63	+ if (a[i] === undefined) return false;
	64	+ const re = new RegExp(
	65	+ "^" + p[i].replace(/[.+?^${}()\|[\]\\]/g, "\\$&").replace(/\/g, ".") + "$",
	66	+ );
	67	+ if (!re.test(a[i])) return false;
	68	+ }
	69	+ return p.length === a.length;
	70	+};
38	71
39	72	/** A plugin extension capable of resolving a selector (slug or id) to a scope,
40	73	* scoped to the caller. `resolveScope` returns null when nothing matches. */
@@ -66,6 +99,10 @@ export const applyToolkitScope = <TPlugins extends readonly AnyPlugin[]>( · 25 unmodified lines
66	99	scope: ResolvedToolkitScope,
67	100	): Effect.Effect<Executor<TPlugins>, StorageFailure> =>
68	101	Effect.gen(function* () {
	102	+ const blockRules = scope.policies.filter((p) => p.action === "block");
	103	+ const isBlocked = (integration: string, connection: string, name: string) =>
	104	+ blockRules.some((p) => matchPattern(p.pattern, `${integration}.${connection}.${name}`));
	105	+
69	106	const all = yield* base.tools.list();
70	107	const allowed = new Set<string>();
71	108	for (const t of all) {
@@ -75,6 +112,7 @@ export const applyToolkitScope = <TPlugins extends readonly AnyPlugin[]>( · 3 unmodified lines
75	112	allowed.add(String(t.address));
76	113	continue;
77	114	}
	115	+ if (isBlocked(String(t.integration), String(t.connection), String(t.name))) continue;
78	116	const a = accessFor(scope, String(t.integration), String(t.connection));
79	117	if (a === "full" \|\| (a === "read" && t.annotations?.readOnly === true)) {
80	118	allowed.add(String(t.address));

packages/plugins/toolkits/src/server.ts+63−20

@@ -29,6 +29,7 @@ import { · 28 unmodified lines
29	29	ToolkitsApi,
30	30	type CreateToolkitPayload,
31	31	type ToolkitAccess,
	32	+ type ToolkitPolicyAction,
32	33	type ToolkitScope,
33	34	type ToolkitView,
34	35	type UpdateToolkitPayload,
@@ -57,10 +58,17 @@ const ConnectionRow = Schema.Struct({ · 22 unmodified lines
57	58	note: Schema.NullOr(Schema.String),
58	59	});
59	60
	61	+const PolicyRow = Schema.Struct({
	62	+ toolkitId: Schema.String,
	63	+ pattern: Schema.String,
	64	+ action: Schema.String,
	65	+});
	66	+
60	67	const TOOLKITS = definePluginStorageCollection("toolkits", ToolkitRow, { indexes: ["slug"] });
61	68	const CONNECTIONS = definePluginStorageCollection("connections", ConnectionRow, {
62	69	indexes: ["toolkitId"],
63	70	});
	71	+const POLICIES = definePluginStorageCollection("policies", PolicyRow, { indexes: ["toolkitId"] });
64	72
65	73	const newId = Effect.sync(() => crypto.randomUUID());
66	74	const scopeOfOwner = (owner: Owner): ToolkitScope => (owner === ORG ? "workspace" : "personal");
@@ -79,26 +87,31 @@ type ToolkitRowEntry = { · 12 unmodified lines
79	87	const makeToolkitsExtension = (ctx: PluginCtx) => {
80	88	const toolkits = ctx.pluginStorage.collection(TOOLKITS);
81	89	const connections = ctx.pluginStorage.collection(CONNECTIONS);
	90	+ const policies = ctx.pluginStorage.collection(POLICIES);
82	91
83	92	const viewFromRow = (row: ToolkitRowEntry) =>
84		− connections.query({ where: { toolkitId: row.key } }).pipe(
85		− Effect.map(
86		− (rows): ToolkitView => ({
87		− id: row.key,
88		− slug: row.data.slug,
89		− name: row.data.name,
90		− scope: scopeOfOwner(row.owner),
91		− inheritOrgPolicies: row.data.inheritOrgPolicies,
92		− briefing: row.data.briefing,
93		− connections: rows.map((c) => ({
94		− integration: c.data.integration as ToolkitConnectionIntegration,
95		− connection: c.data.connection,
96		− access: c.data.access as ToolkitAccess,
97		− ...(c.data.note == null ? {} : { note: c.data.note }),
98		− })),
99		− }),
100		− ),
101		− );
	93	+ Effect.gen(function* () {
	94	+ const conns = yield* connections.query({ where: { toolkitId: row.key } });
	95	+ const pols = yield* policies.query({ where: { toolkitId: row.key } });
	96	+ return {
	97	+ id: row.key,
	98	+ slug: row.data.slug,
	99	+ name: row.data.name,
	100	+ scope: scopeOfOwner(row.owner),
	101	+ inheritOrgPolicies: row.data.inheritOrgPolicies,
	102	+ briefing: row.data.briefing,
	103	+ connections: conns.map((c) => ({
	104	+ integration: c.data.integration as ToolkitConnectionIntegration,
	105	+ connection: c.data.connection,
	106	+ access: c.data.access as ToolkitAccess,
	107	+ ...(c.data.note == null ? {} : { note: c.data.note }),
	108	+ })),
	109	+ policies: pols.map((p) => ({
	110	+ pattern: p.data.pattern,
	111	+ action: p.data.action as ToolkitPolicyAction,
	112	+ })),
	113	+ } satisfies ToolkitView;
	114	+ });
102	115
103	116	// candidate connections honoring scope: a workspace toolkit may use only
104	117	// org connections; a personal toolkit may use org + the caller's own.
@@ -146,6 +159,23 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { · 41 unmodified lines
146	159	),
147	160	);
148	161
	162	+ const putPolicies = (
	163	+ toolkitId: string,
	164	+ owner: Owner,
	165	+ rules: ReadonlyArray<{ readonly pattern: string; readonly action: string }>,
	166	+ ) =>
	167	+ Effect.forEach(rules, (r) =>
	168	+ newId.pipe(
	169	+ Effect.flatMap((pid) =>
	170	+ policies.put({
	171	+ key: pid,
	172	+ owner,
	173	+ data: { toolkitId, pattern: r.pattern, action: r.action },
	174	+ }),
	175	+ ),
	176	+ ),
	177	+ );
	178	+
149	179	const create = (input: CreateToolkitPayload) =>
150	180	Effect.gen(function* () {
151	181	if (input.scope === "personal" && ctx.owner.subject == null) {
@@ -165,6 +195,7 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { · 13 unmodified lines
165	195	};
166	196	yield* toolkits.put({ key: id, owner, data });
167	197	yield* putConnections(id, owner, entries);
	198	+ yield* putPolicies(id, owner, input.policies ?? []);
168	199	return yield* viewFromRow({ key: id, owner, data });
169	200	});
170	201
@@ -197,6 +228,11 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { · 26 unmodified lines
197	228	);
198	229	yield* putConnections(id, row.owner, patch.connections);
199	230	}
	231	+ if (patch.policies !== undefined) {
	232	+ const existing = yield* policies.query({ where: { toolkitId: id } });
	233	+ yield* Effect.forEach(existing, (p) => policies.remove({ key: p.key, owner: row.owner }));
	234	+ yield* putPolicies(id, row.owner, patch.policies);
	235	+ }
200	236	return yield* viewFromRow({ key: id, owner: row.owner, data });
201	237	});
202	238
@@ -204,8 +240,10 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { · 1 unmodified line
204	240	Effect.gen(function* () {
205	241	const row = yield* toolkits.get({ key: id });
206	242	if (row == null) return yield* new ToolkitNotFound({ id });
207		− const existing = yield* connections.query({ where: { toolkitId: id } });
208		− yield* Effect.forEach(existing, (c) => connections.remove({ key: c.key, owner: row.owner }));
	243	+ const conns = yield* connections.query({ where: { toolkitId: id } });
	244	+ yield* Effect.forEach(conns, (c) => connections.remove({ key: c.key, owner: row.owner }));
	245	+ const pols = yield* policies.query({ where: { toolkitId: id } });
	246	+ yield* Effect.forEach(pols, (p) => policies.remove({ key: p.key, owner: row.owner }));
209	247	yield* toolkits.remove({ key: id, owner: row.owner });
210	248	return { removed: true };
211	249	});
@@ -222,12 +260,17 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { · 10 unmodified lines
222	260	const row = bySlug[0] ?? (yield* toolkits.get({ key: selector }));
223	261	if (row == null) return null;
224	262	const conns = yield* connections.query({ where: { toolkitId: row.key } });
	263	+ const pols = yield* policies.query({ where: { toolkitId: row.key } });
225	264	return {
226	265	entries: conns.map((c) => ({
227	266	integration: c.data.integration,
228	267	connection: c.data.connection,
229	268	access: c.data.access as ToolkitAccess,
230	269	})),
	270	+ policies: pols.map((p) => ({
	271	+ pattern: p.data.pattern,
	272	+ action: p.data.action as ToolkitPolicyAction,
	273	+ })),
231	274	inheritOrgPolicies: row.data.inheritOrgPolicies,
232	275	};
233	276	});

packages/plugins/toolkits/src/shared.ts+15−0

@@ -31,6 +31,18 @@ export const ToolkitConnectionEntry = Schema.Struct({ · 30 unmodified lines
31	31	});
32	32	export type ToolkitConnectionEntry = typeof ToolkitConnectionEntry.Type;
33	33
	34	+// off/read/full is the per-connection access mode; policies are pattern rules
	35	+// layered on top. v1 enforces `block` (the tool is excluded); `approve` /
	36	+// `require_approval` are persisted + briefed (org approval policies still apply).
	37	+export const ToolkitPolicyAction = Schema.Literals(["approve", "require_approval", "block"]);
	38	+export type ToolkitPolicyAction = typeof ToolkitPolicyAction.Type;
	39	+
	40	+export const ToolkitPolicy = Schema.Struct({
	41	+ pattern: Schema.String,
	42	+ action: ToolkitPolicyAction,
	43	+});
	44	+export type ToolkitPolicy = typeof ToolkitPolicy.Type;
	45	+
34	46	export const ToolkitView = Schema.Struct({
35	47	id: Schema.String,
36	48	slug: Schema.String,
@@ -39,6 +51,7 @@ export const ToolkitView = Schema.Struct({ · 2 unmodified lines
39	51	inheritOrgPolicies: Schema.Boolean,
40	52	briefing: Schema.NullOr(Schema.String),
41	53	connections: Schema.Array(ToolkitConnectionEntry),
	54	+ policies: Schema.Array(ToolkitPolicy),
42	55	});
43	56	export type ToolkitView = typeof ToolkitView.Type;
44	57
@@ -49,6 +62,7 @@ export const CreateToolkitPayload = Schema.Struct({ · 4 unmodified lines
49	62	inheritOrgPolicies: Schema.optional(Schema.Boolean),
50	63	briefing: Schema.optional(Schema.String),
51	64	connections: Schema.optional(Schema.Array(ToolkitConnectionEntry)),
	65	+ policies: Schema.optional(Schema.Array(ToolkitPolicy)),
52	66	});
53	67	export type CreateToolkitPayload = typeof CreateToolkitPayload.Type;
54	68
@@ -57,6 +71,7 @@ export const UpdateToolkitPayload = Schema.Struct({ · 2 unmodified lines
57	71	briefing: Schema.optional(Schema.NullOr(Schema.String)),
58	72	inheritOrgPolicies: Schema.optional(Schema.Boolean),
59	73	connections: Schema.optional(Schema.Array(ToolkitConnectionEntry)),
	74	+ policies: Schema.optional(Schema.Array(ToolkitPolicy)),
60	75	});
61	76	export type UpdateToolkitPayload = typeof UpdateToolkitPayload.Type;
62	77