History

Reduce Microsoft Graph full import memory

Rhys Sullivan committed 4 months ago

Commit 12b09311 parent 1f696d02 files · +56 −12Browse files

Unified · Split

2 files
  1. packages/plugins/microsoft/src/sdk/graph.ts +47 −10
  2. packages/plugins/microsoft/src/sdk/plugin.ts +9 −2
packages/plugins/microsoft/src/sdk/graph.ts+47−10
@@ -9,6 +9,7 @@ import { · 8 unmodified lines
99 OpenApiParseError,
1010 type Authentication,
1111 type OpenApiIntegrationConfig,
12+ type ParsedDocument,
1213 } from "@executor-js/plugin-openapi";
1314
1415 import {
@@ -41,6 +42,7 @@ export interface MicrosoftGraphSelectionInput { · 26 unmodified lines
4142
4243 export interface MicrosoftGraphSpecBuild {
4344 readonly specText: string;
45+ readonly parsedDocument?: ParsedDocument;
4446 readonly specUrl: string;
4547 readonly baseUrl?: string;
4648 readonly authorizationUrl: string;
@@ -90,6 +92,8 @@ const MicrosoftGraphIntegrationConfigSchema = Schema.Struct({ · 43 unmodified lines
9092
9193 const decodeMicrosoftConfig = Schema.decodeUnknownOption(MicrosoftGraphIntegrationConfigSchema);
9294
95+type MicrosoftGraphOpenApiDocument = ParsedDocument & Record<string, unknown>;
96+
9397 export const decodeMicrosoftGraphIntegrationConfig = (
9498 value: unknown,
9599 ): MicrosoftGraphIntegrationConfig | null =>
@@ -478,7 +482,7 @@ export const fetchMicrosoftGraphPermissionsReference = Effect.fn( · 382 unmodified lines
478482
479483 const parseMicrosoftGraphOpenApiDocument = (
480484 specText: string,
481−): Effect.Effect<Record<string, unknown>, OpenApiParseError> =>
485+): Effect.Effect<MicrosoftGraphOpenApiDocument, OpenApiParseError> =>
482486 Effect.gen(function* () {
483487 const parsed = yield* Effect.try({
484488 try: () => YAML.parse(specText) as unknown,
@@ -492,11 +496,16 @@ const parseMicrosoftGraphOpenApiDocument = ( · 7 unmodified lines
492496 message: "Microsoft Graph OpenAPI document must be an object",
493497 });
494498 }
495− return parsed;
499+ if (typeof parsed.openapi !== "string" || !parsed.openapi.startsWith("3.")) {
500+ return yield* new OpenApiParseError({
501+ message: "Microsoft Graph OpenAPI document must be OpenAPI 3.x",
502+ });
503+ }
504+ return parsed as MicrosoftGraphOpenApiDocument;
496505 });
497506
498−export const buildFilteredMicrosoftGraphOpenApiSpec = (
499− specText: string,
507+export const buildFilteredMicrosoftGraphOpenApiSpecFromDocument = (
508+ parsed: MicrosoftGraphOpenApiDocument,
500509 options: {
501510 readonly scopes: readonly string[];
502511 readonly exactPaths: readonly string[];
@@ -510,7 +519,6 @@ export const buildFilteredMicrosoftGraphOpenApiSpec = ( · 7 unmodified lines
510519 },
511520 ): Effect.Effect<MicrosoftGraphFilterResult, OpenApiParseError> =>
512521 Effect.gen(function* () {
513− const parsed = yield* parseMicrosoftGraphOpenApiDocument(specText);
514522 const paths = parsed.paths;
515523 if (!isRecord(paths)) {
516524 return yield* new OpenApiParseError({
@@ -578,6 +586,15 @@ export const buildFilteredMicrosoftGraphOpenApiSpec = ( · 61 unmodified lines
578586 return { specText: filteredSpecText, scopes };
579587 });
580588
589+export const buildFilteredMicrosoftGraphOpenApiSpec = (
590+ specText: string,
591+ options: Parameters<typeof buildFilteredMicrosoftGraphOpenApiSpecFromDocument>[1],
592+): Effect.Effect<MicrosoftGraphFilterResult, OpenApiParseError> =>
593+ Effect.gen(function* () {
594+ const parsed = yield* parseMicrosoftGraphOpenApiDocument(specText);
595+ return yield* buildFilteredMicrosoftGraphOpenApiSpecFromDocument(parsed, options);
596+ });
597+
581598 export const filterMicrosoftGraphOpenApiSpec = (
582599 specText: string,
583600 options: Parameters<typeof buildFilteredMicrosoftGraphOpenApiSpec>[1],
@@ -604,12 +621,32 @@ export const buildMicrosoftGraphOpenApiSpec = ( · 20 unmodified lines
604621 : [];
605622 const parsed = yield* parseMicrosoftGraphOpenApiDocument(sourceText);
606623 const endpoints = resolveOAuthEndpoints(parsed, selection);
607− const filtered = yield* buildFilteredMicrosoftGraphOpenApiSpec(sourceText, {
624+ const graphPaths = parsed.paths;
625+ if (!isRecord(graphPaths)) {
626+ return yield* new OpenApiParseError({
627+ message: "Microsoft Graph OpenAPI document is missing paths",
628+ });
629+ }
630+ if (selection.coversFullGraph === true) {
631+ const scopes = selectedOAuthScopesForPaths(
632+ graphPaths,
633+ uniqueStrings([...MICROSOFT_GRAPH_BASE_SCOPES, ...selection.customScopes]),
634+ fullGraphScopes,
635+ );
636+ return {
637+ ...selection,
638+ specText: sourceText,
639+ parsedDocument: parsed,
640+ scopes,
641+ authorizationUrl: endpoints.authorizationUrl,
642+ tokenUrl: endpoints.tokenUrl,
643+ clientCredentialsTokenUrl: endpoints.clientCredentialsTokenUrl,
644+ authenticationTemplate: microsoftOAuthTemplate(scopes, endpoints),
645+ };
646+ }
647+ const filtered = yield* buildFilteredMicrosoftGraphOpenApiSpecFromDocument(parsed, {
608648 ...selection,
609− scopes:
610− selection.coversFullGraph === true
611− ? uniqueStrings([...MICROSOFT_GRAPH_BASE_SCOPES, ...selection.customScopes])
612− : selection.scopes,
649+ scopes: selection.scopes,
613650 fullGraphScopes,
614651 });
615652 return {
packages/plugins/microsoft/src/sdk/plugin.ts+9−2
@@ -18,6 +18,7 @@ import { · 17 unmodified lines
1818 } from "@executor-js/sdk/core";
1919 import { describeApiKeyAuthMethod } from "@executor-js/sdk/http-auth";
2020 import {
21+ compileOpenApiDocument,
2122 compileOpenApiSpec,
2223 decodeOpenApiIntegrationConfig,
2324 invokeOpenApiBackedTool,
@@ -124,7 +125,10 @@ const makeMicrosoftPluginExtension = ( · 100 unmodified lines
124125 const addGraph = (config: MicrosoftGraphConfig) =>
125126 Effect.gen(function* () {
126127 const graph = yield* buildMicrosoftGraphOpenApiSpec(config, httpClientLayer);
127− const compiled = yield* compileOpenApiSpec(graph.specText);
128+ const compiled =
129+ graph.parsedDocument !== undefined
130+ ? yield* compileOpenApiDocument(graph.parsedDocument)
131+ : yield* compileOpenApiSpec(graph.specText);
128132 const slug = IntegrationSlug.make(config.slug?.trim() || DEFAULT_MICROSOFT_SLUG);
129133
130134 const existing = yield* ctx.core.integrations.get(slug);
@@ -195,7 +199,10 @@ const makeMicrosoftPluginExtension = ( · 64 unmodified lines
195199 },
196200 httpClientLayer,
197201 );
198− const compiled = yield* compileOpenApiSpec(graph.specText);
202+ const compiled =
203+ graph.parsedDocument !== undefined
204+ ? yield* compileOpenApiDocument(graph.parsedDocument)
205+ : yield* compileOpenApiSpec(graph.specText);
199206
200207 const previousOperations = yield* ctx.storage.listOperations(rawSlug);
201208 const previousNames = new Set(previousOperations.map((op) => op.toolName));