History

feat(toolkits): per-toolkit policies + access-mode enforcement (slice 4)

Add a toolkit_policies collection (pattern -> approve|require_approval|block) to the plugin + CRUD/view. resolveScope returns the policies; the Executor wrapper enforces `block` as exclusion (the tool is removed from the slice and blocked at execute). Read-only access filters to tools annotated readOnly (fail-closed: unclassified counts as write). approve/require_approval are persisted + surfaced; org-level approval policies still apply via the base executor (toolkit-level approval enforcement is a follow-up). e2e (selfhost): full runs the tool; a block policy stops it; read-only access blocks an unclassified tool.

Rhys Sullivan committed 4 months ago

Commit d059ee81 parent 2f9241d5 files · +240 −21Browse files

Unified · Split

5 files
  1. e2e/selfhost/toolkits-policies.test.ts +121 −0
  2. packages/core/sdk/src/index.ts +2 −0
  3. packages/core/sdk/src/toolkit-scope.ts +39 −1
  4. packages/plugins/toolkits/src/server.ts +63 −20
  5. packages/plugins/toolkits/src/shared.ts +15 −0
e2e/selfhost/toolkits-policies.test.ts+121−0
@@ -0,0 +1,121 @@
1// Selfhost · Slice 4: per-toolkit policies + access modes at execute time.
2// - full access -> the connection's tool runs;
3// - a `block` policy on that tool -> blocked at execute (exclusion);
4// - read-only access -> an unclassified (non-read-only) tool is blocked
5// (fail-closed: unclassified counts as write).
6// One greeting connection, three toolkits over it, compared via MCP execute.
7import { randomBytes } from "node:crypto";
8
9import { expect } from "@effect/vitest";
10import { Effect } from "effect";
11import { composePluginApi } from "@executor-js/api/server";
12import { mcpHttpPlugin } from "@executor-js/plugin-mcp/api";
13import { makeGreetingMcpServer, serveMcpServer } from "@executor-js/plugin-mcp/testing";
14import { toolkitsPlugin } from "@executor-js/plugin-toolkits/server";
15import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk/shared";
16
17import { scenario } from "../src/scenario";
18import { Api, Mcp, Target } from "../src/services";
19
20const api = composePluginApi([mcpHttpPlugin(), toolkitsPlugin()] as const);
21const ident = (prefix: string): string => `${prefix}${randomBytes(4).toString("hex")}`;
22
23scenario(
24 "Toolkits · access modes + block policies are enforced at execute",
25 { timeout: 180_000 },
26 Effect.scoped(
27 Effect.gen(function* () {
28 const target = yield* Target;
29 const { client: makeApiClient } = yield* Api;
30 const mcp = yield* Mcp;
31 const identity = yield* target.newIdentity();
32 const client = yield* makeApiClient(api, identity);
33
34 const slug = ident("pol");
35 const conn = ident("conn");
36 const token = `tok-${randomBytes(6).toString("hex")}`;
37 const server = yield* serveMcpServer(() => makeGreetingMcpServer(), {
38 auth: {
39 validateAuthorization: (authorization) =>
40 Effect.succeed(authorization === `Bearer ${token}`),
41 },
42 });
43 yield* client.mcp.addServer({
44 payload: {
45 transport: "remote",
46 name: `Greeting ${slug}`,
47 endpoint: server.endpoint,
48 slug,
49 authenticationTemplate: [
50 {
51 type: "apiKey",
52 headers: { Authorization: ["Bearer ", { type: "variable", name: "token" }] },
53 },
54 ],
55 },
56 });
57 yield* client.connections.create({
58 payload: {
59 owner: "org",
60 name: ConnectionName.make(conn),
61 integration: IntegrationSlug.make(slug),
62 template: AuthTemplateSlug.make("header"),
63 value: token,
64 },
65 });
66
67 const full = yield* client.toolkits.create({
68 payload: {
69 slug: ident("kitfull"),
70 name: "Full",
71 scope: "workspace",
72 connections: [
73 { integration: IntegrationSlug.make(slug), connection: conn, access: "full" },
74 ],
75 },
76 });
77 const blocked = yield* client.toolkits.create({
78 payload: {
79 slug: ident("kitblock"),
80 name: "Blocked",
81 scope: "workspace",
82 connections: [
83 { integration: IntegrationSlug.make(slug), connection: conn, access: "full" },
84 ],
85 policies: [{ pattern: `${slug}.${conn}.simple_echo`, action: "block" }],
86 },
87 });
88 const readonly = yield* client.toolkits.create({
89 payload: {
90 slug: ident("kitread"),
91 name: "Read",
92 scope: "workspace",
93 connections: [
94 { integration: IntegrationSlug.make(slug), connection: conn, access: "read" },
95 ],
96 },
97 });
98
99 const code = `return await tools.${slug}.org.${conn}.simple_echo({});`;
100 const fullCall = yield* mcp
101 .session(identity, { toolkit: full.slug })
102 .call("execute", { code });
103 const blockCall = yield* mcp
104 .session(identity, { toolkit: blocked.slug })
105 .call("execute", { code });
106 const readCall = yield* mcp
107 .session(identity, { toolkit: readonly.slug })
108 .call("execute", { code });
109
110 expect(fullCall.ok, `full access runs the tool; text=${fullCall.text}`).toBe(true);
111 expect(
112 blockCall.text,
113 `block policy must stop the tool; full=${fullCall.text} block=${blockCall.text}`,
114 ).not.toBe(fullCall.text);
115 expect(
116 readCall.text,
117 `read-only must hide an unclassified (write) tool; full=${fullCall.text} read=${readCall.text}`,
118 ).not.toBe(fullCall.text);
119 }),
120 ),
121);
packages/core/sdk/src/index.ts+2−0
@@ -358,6 +358,8 @@ export { · 357 unmodified lines
358 EMPTY_TOOLKIT_SCOPE,358 EMPTY_TOOLKIT_SCOPE,
359 type ResolvedToolkitScope,359 type ResolvedToolkitScope,
360 type ToolkitAccess,360 type ToolkitAccess,
361 type ToolkitPolicyAction,
362 type ToolkitPolicyRule,
361 type ToolkitResolver,363 type ToolkitResolver,
362 type ToolkitScopeEntry,364 type ToolkitScopeEntry,
363} from "./toolkit-scope";365} from "./toolkit-scope";
packages/core/sdk/src/toolkit-scope.ts+39−1
@@ -19,6 +19,15 @@ import type { StorageFailure } from "./fuma-runtime"; · 18 unmodified lines
19 19
20export type ToolkitAccess = "off" | "read" | "full";20export type ToolkitAccess = "off" | "read" | "full";
21 21
22export type ToolkitPolicyAction = "approve" | "require_approval" | "block";
23
24export interface ToolkitPolicyRule {
25 /** Glob over `<integration>.<connection>.<tool>` — `*` matches one segment,
26 * a trailing `*` matches the rest, and `*` inside a segment globs. */
27 readonly pattern: string;
28 readonly action: ToolkitPolicyAction;
29}
30
22export interface ToolkitScopeEntry {31export interface ToolkitScopeEntry {
23 readonly integration: string;32 readonly integration: string;
24 /** A pinned connection name, or "*" to track every connection of the integration. */33 /** A pinned connection name, or "*" to track every connection of the integration. */
@@ -28,13 +37,37 @@ export interface ToolkitScopeEntry { · 3 unmodified lines
28 37
29export interface ResolvedToolkitScope {38export interface ResolvedToolkitScope {
30 readonly entries: readonly ToolkitScopeEntry[];39 readonly entries: readonly ToolkitScopeEntry[];
40 /** Per-toolkit policy rules. v1 enforces `block` (exclusion); `approve` /
41 * `require_approval` are carried for the briefing — org-level approval
42 * policies are still enforced by the base executor when inherited. */
43 readonly policies: readonly ToolkitPolicyRule[];
31 readonly inheritOrgPolicies: boolean;44 readonly inheritOrgPolicies: boolean;
32}45}
33 46
34/** Empty slice — exposes no connection tools (static/core tools stay visible).47/** Empty slice — exposes no connection tools (static/core tools stay visible).
35 * Applied fail-closed when a selector resolves to no toolkit (unknown or48 * Applied fail-closed when a selector resolves to no toolkit (unknown or
36 * not visible to the caller), so a bad selector never leaks the full account. */49 * not visible to the caller), so a bad selector never leaks the full account. */
37export const EMPTY_TOOLKIT_SCOPE: ResolvedToolkitScope = { entries: [], inheritOrgPolicies: true };50export const EMPTY_TOOLKIT_SCOPE: ResolvedToolkitScope = {
51 entries: [],
52 policies: [],
53 inheritOrgPolicies: true,
54};
55
56/** Glob match over dot segments: `*` matches one segment, a trailing bare `*`
57 * matches the rest, and `*` inside a segment globs within it. */
58const matchPattern = (pattern: string, target: string): boolean => {
59 const p = pattern.toLowerCase().trim().split(".");
60 const a = target.toLowerCase().split(".");
61 for (let i = 0; i < p.length; i++) {
62 if (p[i] === "*" && i === p.length - 1) return true;
63 if (a[i] === undefined) return false;
64 const re = new RegExp(
65 "^" + p[i].replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*") + "$",
66 );
67 if (!re.test(a[i])) return false;
68 }
69 return p.length === a.length;
70};
38 71
39/** A plugin extension capable of resolving a selector (slug or id) to a scope,72/** A plugin extension capable of resolving a selector (slug or id) to a scope,
40 * scoped to the caller. `resolveScope` returns null when nothing matches. */73 * scoped to the caller. `resolveScope` returns null when nothing matches. */
@@ -66,6 +99,10 @@ export const applyToolkitScope = <TPlugins extends readonly AnyPlugin[]>( · 25 unmodified lines
66 scope: ResolvedToolkitScope,99 scope: ResolvedToolkitScope,
67): Effect.Effect<Executor<TPlugins>, StorageFailure> =>100): Effect.Effect<Executor<TPlugins>, StorageFailure> =>
68 Effect.gen(function* () {101 Effect.gen(function* () {
102 const blockRules = scope.policies.filter((p) => p.action === "block");
103 const isBlocked = (integration: string, connection: string, name: string) =>
104 blockRules.some((p) => matchPattern(p.pattern, `${integration}.${connection}.${name}`));
105
69 const all = yield* base.tools.list();106 const all = yield* base.tools.list();
70 const allowed = new Set<string>();107 const allowed = new Set<string>();
71 for (const t of all) {108 for (const t of all) {
@@ -75,6 +112,7 @@ export const applyToolkitScope = <TPlugins extends readonly AnyPlugin[]>( · 3 unmodified lines
75 allowed.add(String(t.address));112 allowed.add(String(t.address));
76 continue;113 continue;
77 }114 }
115 if (isBlocked(String(t.integration), String(t.connection), String(t.name))) continue;
78 const a = accessFor(scope, String(t.integration), String(t.connection));116 const a = accessFor(scope, String(t.integration), String(t.connection));
79 if (a === "full" || (a === "read" && t.annotations?.readOnly === true)) {117 if (a === "full" || (a === "read" && t.annotations?.readOnly === true)) {
80 allowed.add(String(t.address));118 allowed.add(String(t.address));
packages/plugins/toolkits/src/server.ts+63−20
@@ -29,6 +29,7 @@ import { · 28 unmodified lines
29 ToolkitsApi,29 ToolkitsApi,
30 type CreateToolkitPayload,30 type CreateToolkitPayload,
31 type ToolkitAccess,31 type ToolkitAccess,
32 type ToolkitPolicyAction,
32 type ToolkitScope,33 type ToolkitScope,
33 type ToolkitView,34 type ToolkitView,
34 type UpdateToolkitPayload,35 type UpdateToolkitPayload,
@@ -57,10 +58,17 @@ const ConnectionRow = Schema.Struct({ · 22 unmodified lines
57 note: Schema.NullOr(Schema.String),58 note: Schema.NullOr(Schema.String),
58});59});
59 60
61const PolicyRow = Schema.Struct({
62 toolkitId: Schema.String,
63 pattern: Schema.String,
64 action: Schema.String,
65});
66
60const TOOLKITS = definePluginStorageCollection("toolkits", ToolkitRow, { indexes: ["slug"] });67const TOOLKITS = definePluginStorageCollection("toolkits", ToolkitRow, { indexes: ["slug"] });
61const CONNECTIONS = definePluginStorageCollection("connections", ConnectionRow, {68const CONNECTIONS = definePluginStorageCollection("connections", ConnectionRow, {
62 indexes: ["toolkitId"],69 indexes: ["toolkitId"],
63});70});
71const POLICIES = definePluginStorageCollection("policies", PolicyRow, { indexes: ["toolkitId"] });
64 72
65const newId = Effect.sync(() => crypto.randomUUID());73const newId = Effect.sync(() => crypto.randomUUID());
66const scopeOfOwner = (owner: Owner): ToolkitScope => (owner === ORG ? "workspace" : "personal");74const scopeOfOwner = (owner: Owner): ToolkitScope => (owner === ORG ? "workspace" : "personal");
@@ -79,26 +87,31 @@ type ToolkitRowEntry = { · 12 unmodified lines
79const makeToolkitsExtension = (ctx: PluginCtx) => {87const makeToolkitsExtension = (ctx: PluginCtx) => {
80 const toolkits = ctx.pluginStorage.collection(TOOLKITS);88 const toolkits = ctx.pluginStorage.collection(TOOLKITS);
81 const connections = ctx.pluginStorage.collection(CONNECTIONS);89 const connections = ctx.pluginStorage.collection(CONNECTIONS);
90 const policies = ctx.pluginStorage.collection(POLICIES);
82 91
83 const viewFromRow = (row: ToolkitRowEntry) =>92 const viewFromRow = (row: ToolkitRowEntry) =>
84 connections.query({ where: { toolkitId: row.key } }).pipe(93 Effect.gen(function* () {
85 Effect.map(94 const conns = yield* connections.query({ where: { toolkitId: row.key } });
86 (rows): ToolkitView => ({95 const pols = yield* policies.query({ where: { toolkitId: row.key } });
87 id: row.key,96 return {
88 slug: row.data.slug,97 id: row.key,
89 name: row.data.name,98 slug: row.data.slug,
90 scope: scopeOfOwner(row.owner),99 name: row.data.name,
91 inheritOrgPolicies: row.data.inheritOrgPolicies,100 scope: scopeOfOwner(row.owner),
92 briefing: row.data.briefing,101 inheritOrgPolicies: row.data.inheritOrgPolicies,
93 connections: rows.map((c) => ({102 briefing: row.data.briefing,
94 integration: c.data.integration as ToolkitConnectionIntegration,103 connections: conns.map((c) => ({
95 connection: c.data.connection,104 integration: c.data.integration as ToolkitConnectionIntegration,
96 access: c.data.access as ToolkitAccess,105 connection: c.data.connection,
97 ...(c.data.note == null ? {} : { note: c.data.note }),106 access: c.data.access as ToolkitAccess,
98 })),107 ...(c.data.note == null ? {} : { note: c.data.note }),
99 }),108 })),
100 ),109 policies: pols.map((p) => ({
101 );110 pattern: p.data.pattern,
111 action: p.data.action as ToolkitPolicyAction,
112 })),
113 } satisfies ToolkitView;
114 });
102 115
103 // candidate connections honoring scope: a workspace toolkit may use only116 // candidate connections honoring scope: a workspace toolkit may use only
104 // org connections; a personal toolkit may use org + the caller's own.117 // org connections; a personal toolkit may use org + the caller's own.
@@ -146,6 +159,23 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { · 41 unmodified lines
146 ),159 ),
147 );160 );
148 161
162 const putPolicies = (
163 toolkitId: string,
164 owner: Owner,
165 rules: ReadonlyArray<{ readonly pattern: string; readonly action: string }>,
166 ) =>
167 Effect.forEach(rules, (r) =>
168 newId.pipe(
169 Effect.flatMap((pid) =>
170 policies.put({
171 key: pid,
172 owner,
173 data: { toolkitId, pattern: r.pattern, action: r.action },
174 }),
175 ),
176 ),
177 );
178
149 const create = (input: CreateToolkitPayload) =>179 const create = (input: CreateToolkitPayload) =>
150 Effect.gen(function* () {180 Effect.gen(function* () {
151 if (input.scope === "personal" && ctx.owner.subject == null) {181 if (input.scope === "personal" && ctx.owner.subject == null) {
@@ -165,6 +195,7 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { · 13 unmodified lines
165 };195 };
166 yield* toolkits.put({ key: id, owner, data });196 yield* toolkits.put({ key: id, owner, data });
167 yield* putConnections(id, owner, entries);197 yield* putConnections(id, owner, entries);
198 yield* putPolicies(id, owner, input.policies ?? []);
168 return yield* viewFromRow({ key: id, owner, data });199 return yield* viewFromRow({ key: id, owner, data });
169 });200 });
170 201
@@ -197,6 +228,11 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { · 26 unmodified lines
197 );228 );
198 yield* putConnections(id, row.owner, patch.connections);229 yield* putConnections(id, row.owner, patch.connections);
199 }230 }
231 if (patch.policies !== undefined) {
232 const existing = yield* policies.query({ where: { toolkitId: id } });
233 yield* Effect.forEach(existing, (p) => policies.remove({ key: p.key, owner: row.owner }));
234 yield* putPolicies(id, row.owner, patch.policies);
235 }
200 return yield* viewFromRow({ key: id, owner: row.owner, data });236 return yield* viewFromRow({ key: id, owner: row.owner, data });
201 });237 });
202 238
@@ -204,8 +240,10 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { · 1 unmodified line
204 Effect.gen(function* () {240 Effect.gen(function* () {
205 const row = yield* toolkits.get({ key: id });241 const row = yield* toolkits.get({ key: id });
206 if (row == null) return yield* new ToolkitNotFound({ id });242 if (row == null) return yield* new ToolkitNotFound({ id });
207 const existing = yield* connections.query({ where: { toolkitId: id } });243 const conns = yield* connections.query({ where: { toolkitId: id } });
208 yield* Effect.forEach(existing, (c) => connections.remove({ key: c.key, owner: row.owner }));244 yield* Effect.forEach(conns, (c) => connections.remove({ key: c.key, owner: row.owner }));
245 const pols = yield* policies.query({ where: { toolkitId: id } });
246 yield* Effect.forEach(pols, (p) => policies.remove({ key: p.key, owner: row.owner }));
209 yield* toolkits.remove({ key: id, owner: row.owner });247 yield* toolkits.remove({ key: id, owner: row.owner });
210 return { removed: true };248 return { removed: true };
211 });249 });
@@ -222,12 +260,17 @@ const makeToolkitsExtension = (ctx: PluginCtx) => { · 10 unmodified lines
222 const row = bySlug[0] ?? (yield* toolkits.get({ key: selector }));260 const row = bySlug[0] ?? (yield* toolkits.get({ key: selector }));
223 if (row == null) return null;261 if (row == null) return null;
224 const conns = yield* connections.query({ where: { toolkitId: row.key } });262 const conns = yield* connections.query({ where: { toolkitId: row.key } });
263 const pols = yield* policies.query({ where: { toolkitId: row.key } });
225 return {264 return {
226 entries: conns.map((c) => ({265 entries: conns.map((c) => ({
227 integration: c.data.integration,266 integration: c.data.integration,
228 connection: c.data.connection,267 connection: c.data.connection,
229 access: c.data.access as ToolkitAccess,268 access: c.data.access as ToolkitAccess,
230 })),269 })),
270 policies: pols.map((p) => ({
271 pattern: p.data.pattern,
272 action: p.data.action as ToolkitPolicyAction,
273 })),
231 inheritOrgPolicies: row.data.inheritOrgPolicies,274 inheritOrgPolicies: row.data.inheritOrgPolicies,
232 };275 };
233 });276 });
packages/plugins/toolkits/src/shared.ts+15−0
@@ -31,6 +31,18 @@ export const ToolkitConnectionEntry = Schema.Struct({ · 30 unmodified lines
31});31});
32export type ToolkitConnectionEntry = typeof ToolkitConnectionEntry.Type;32export type ToolkitConnectionEntry = typeof ToolkitConnectionEntry.Type;
33 33
34// off/read/full is the per-connection access mode; policies are pattern rules
35// layered on top. v1 enforces `block` (the tool is excluded); `approve` /
36// `require_approval` are persisted + briefed (org approval policies still apply).
37export const ToolkitPolicyAction = Schema.Literals(["approve", "require_approval", "block"]);
38export type ToolkitPolicyAction = typeof ToolkitPolicyAction.Type;
39
40export const ToolkitPolicy = Schema.Struct({
41 pattern: Schema.String,
42 action: ToolkitPolicyAction,
43});
44export type ToolkitPolicy = typeof ToolkitPolicy.Type;
45
34export const ToolkitView = Schema.Struct({46export const ToolkitView = Schema.Struct({
35 id: Schema.String,47 id: Schema.String,
36 slug: Schema.String,48 slug: Schema.String,
@@ -39,6 +51,7 @@ export const ToolkitView = Schema.Struct({ · 2 unmodified lines
39 inheritOrgPolicies: Schema.Boolean,51 inheritOrgPolicies: Schema.Boolean,
40 briefing: Schema.NullOr(Schema.String),52 briefing: Schema.NullOr(Schema.String),
41 connections: Schema.Array(ToolkitConnectionEntry),53 connections: Schema.Array(ToolkitConnectionEntry),
54 policies: Schema.Array(ToolkitPolicy),
42});55});
43export type ToolkitView = typeof ToolkitView.Type;56export type ToolkitView = typeof ToolkitView.Type;
44 57
@@ -49,6 +62,7 @@ export const CreateToolkitPayload = Schema.Struct({ · 4 unmodified lines
49 inheritOrgPolicies: Schema.optional(Schema.Boolean),62 inheritOrgPolicies: Schema.optional(Schema.Boolean),
50 briefing: Schema.optional(Schema.String),63 briefing: Schema.optional(Schema.String),
51 connections: Schema.optional(Schema.Array(ToolkitConnectionEntry)),64 connections: Schema.optional(Schema.Array(ToolkitConnectionEntry)),
65 policies: Schema.optional(Schema.Array(ToolkitPolicy)),
52});66});
53export type CreateToolkitPayload = typeof CreateToolkitPayload.Type;67export type CreateToolkitPayload = typeof CreateToolkitPayload.Type;
54 68
@@ -57,6 +71,7 @@ export const UpdateToolkitPayload = Schema.Struct({ · 2 unmodified lines
57 briefing: Schema.optional(Schema.NullOr(Schema.String)),71 briefing: Schema.optional(Schema.NullOr(Schema.String)),
58 inheritOrgPolicies: Schema.optional(Schema.Boolean),72 inheritOrgPolicies: Schema.optional(Schema.Boolean),
59 connections: Schema.optional(Schema.Array(ToolkitConnectionEntry)),73 connections: Schema.optional(Schema.Array(ToolkitConnectionEntry)),
74 policies: Schema.optional(Schema.Array(ToolkitPolicy)),
60});75});
61export type UpdateToolkitPayload = typeof UpdateToolkitPayload.Type;76export type UpdateToolkitPayload = typeof UpdateToolkitPayload.Type;
62 77