History

Merge pull request #730 from RhysSullivan/codex/remove-openapi-import-limits

[codex] Remove OpenAPI import size limits

Rhys Sullivan committed 6 months ago

Commit c4891d22 parents a21e52c b1066307 files · +9 −182Browse files

Unified · Split

7 files
  1. apps/cloud/src/mcp.ts +2 −24
  2. apps/cloud/src/sentry-tunnel.ts +2 −31
  3. apps/cloud/src/start.test.ts +0 −21
  4. packages/core/sdk/src/executor.ts +1 −1
  5. packages/core/sdk/src/hosted-http-client.test.ts +0 −40
  6. packages/core/sdk/src/hosted-http-client.ts +2 −63
  7. packages/plugins/openapi/src/sdk/parse.ts +2 −2
apps/cloud/src/mcp.ts+2−24
@@ -72,7 +72,6 @@ const MCP_PATH = "/mcp"; · 71 unmodified lines
72const PROTECTED_RESOURCE_METADATA_PATH = "/.well-known/oauth-protected-resource/mcp";72const PROTECTED_RESOURCE_METADATA_PATH = "/.well-known/oauth-protected-resource/mcp";
73const PROTECTED_RESOURCE_METADATA_URL = `${RESOURCE_ORIGIN}${PROTECTED_RESOURCE_METADATA_PATH}`;73const PROTECTED_RESOURCE_METADATA_URL = `${RESOURCE_ORIGIN}${PROTECTED_RESOURCE_METADATA_PATH}`;
74const RESOURCE_URL = `${RESOURCE_ORIGIN}${MCP_PATH}`;74const RESOURCE_URL = `${RESOURCE_ORIGIN}${MCP_PATH}`;
75export const MAX_MCP_JSON_RPC_BODY_BYTES = 1_000_000;
76 75
77type McpUnauthorizedReason = "missing_bearer" | "invalid_token";76type McpUnauthorizedReason = "missing_bearer" | "invalid_token";
78 77
@@ -108,18 +107,6 @@ const corsPreflight = HttpServerResponse.empty({ · 29 unmodified lines
108 headers: CORS_PREFLIGHT_HEADERS,107 headers: CORS_PREFLIGHT_HEADERS,
109});108});
110 109
111const contentLength = (request: Request): number | null => {
112 const value = request.headers.get("content-length");
113 if (!value) return null;
114 const parsed = Number(value);
115 return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
116};
117
118const isBodyTooLarge = (request: Request, maxBytes: number): boolean => {
119 const length = contentLength(request);
120 return length !== null && length > maxBytes;
121};
122
123// ---------------------------------------------------------------------------110// ---------------------------------------------------------------------------
124// Auth111// Auth
125// ---------------------------------------------------------------------------112// ---------------------------------------------------------------------------
@@ -315,11 +302,7 @@ const readJsonRpcEnvelope = (request: Request): Effect.Effect<Option.Option<Json · 189 unmodified lines
315 try: () => request.clone().text(),302 try: () => request.clone().text(),
316 catch: () => undefined,303 catch: () => undefined,
317 }).pipe(304 }).pipe(
318 Effect.map((text) =>305 Effect.map((text) => (text ? decodeJsonRpcEnvelopeString(text) : Option.none())),
319 text && text.length <= MAX_MCP_JSON_RPC_BODY_BYTES
320 ? decodeJsonRpcEnvelopeString(text)
321 : Option.none(),
322 ),
323 Effect.catchCause(() => Effect.succeed(Option.none())),306 Effect.catchCause(() => Effect.succeed(Option.none())),
324 Effect.withSpan("mcp.request.read_json_rpc"),307 Effect.withSpan("mcp.request.read_json_rpc"),
325 );308 );
@@ -691,23 +674,18 @@ export const mcpApp: Effect.Effect< · 365 unmodified lines
691 return yield* authTemporarilyUnavailable(authResult.failure);674 return yield* authTemporarilyUnavailable(authResult.failure);
692 }675 }
693 const authValue = authResult.success;676 const authValue = authResult.success;
694 const bodyTooLarge =
695 request.method === "POST" && isBodyTooLarge(request, MAX_MCP_JSON_RPC_BODY_BYTES);
696 677
697 // Annotate before dispatch so even 401s show up with what we know. Only678 // Annotate before dispatch so even 401s show up with what we know. Only
698 // POST bodies are JSON-RPC payloads worth parsing; GET (SSE) and DELETE679 // POST bodies are JSON-RPC payloads worth parsing; GET (SSE) and DELETE
699 // don't carry one.680 // don't carry one.
700 yield* annotateMcpRequest(request, {681 yield* annotateMcpRequest(request, {
701 token: isMcpAuthorized(authValue) ? authValue.token : null,682 token: isMcpAuthorized(authValue) ? authValue.token : null,
702 parseBody: request.method === "POST" && isMcpAuthorized(authValue) && !bodyTooLarge,683 parseBody: request.method === "POST" && isMcpAuthorized(authValue),
703 });684 });
704 685
705 if (isMcpUnauthorized(authValue)) {686 if (isMcpUnauthorized(authValue)) {
706 return unauthorized(authValue, PROTECTED_RESOURCE_METADATA_URL);687 return unauthorized(authValue, PROTECTED_RESOURCE_METADATA_URL);
707 }688 }
708 if (bodyTooLarge) {
709 return jsonRpcError(413, -32000, "Request body too large");
710 }
711 const token = authValue.token;689 const token = authValue.token;
712 switch (request.method) {690 switch (request.method) {
713 case "POST":691 case "POST":
apps/cloud/src/sentry-tunnel.ts+2−31
@@ -1,7 +1,5 @@
1import { Data, Effect, Schema } from "effect";1import { Data, Effect, Schema } from "effect";
2 2
3export const MAX_SENTRY_ENVELOPE_BODY_BYTES = 200_000;
4
5class SentryTunnelError extends Data.TaggedError("SentryTunnelError")<{3class SentryTunnelError extends Data.TaggedError("SentryTunnelError")<{
6 readonly cause?: unknown;4 readonly cause?: unknown;
7}> {}5}> {}
@@ -16,39 +14,12 @@ const decodeSentryEnvelopeHeader = Schema.decodeUnknownEffect( · 8 unmodified lines
16 14
17const badSentryEnvelopeResponse = () => new Response("bad envelope", { status: 400 });15const badSentryEnvelopeResponse = () => new Response("bad envelope", { status: 400 });
18 16
19const requestContentLength = (request: Request): number | null => {17export const handleSentryTunnelRequest = (request: Request, configuredDsn: string) =>
20 const value = request.headers.get("content-length");
21 if (!value) return null;
22 const parsed = Number(value);
23 return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
24};
25
26const readLimitedRequestText = (request: Request, maxBytes: number) =>
27 Effect.gen(function* () {18 Effect.gen(function* () {
28 const length = requestContentLength(request);19 const envelope = yield* Effect.tryPromise({
29 if (length !== null && length > maxBytes) {
30 return yield* new SentryTunnelError({ cause: "payload_too_large" });
31 }
32 const text = yield* Effect.tryPromise({
33 try: () => request.text(),20 try: () => request.text(),
34 catch: (cause) => new SentryTunnelError({ cause }),21 catch: (cause) => new SentryTunnelError({ cause }),
35 });22 });
36 if (text.length > maxBytes) {
37 return yield* new SentryTunnelError({ cause: "payload_too_large" });
38 }
39 return text;
40 });
41
42export const handleSentryTunnelRequest = (request: Request, configuredDsn: string) =>
43 Effect.gen(function* () {
44 const envelope = yield* readLimitedRequestText(request, MAX_SENTRY_ENVELOPE_BODY_BYTES).pipe(
45 Effect.catchTag("SentryTunnelError", (error) =>
46 error.cause === "payload_too_large"
47 ? Effect.succeed(new Response("payload too large", { status: 413 }))
48 : Effect.fail(error),
49 ),
50 );
51 if (envelope instanceof Response) return envelope;
52 const firstLine = envelope.slice(0, envelope.indexOf("\n"));23 const firstLine = envelope.slice(0, envelope.indexOf("\n"));
53 const header = yield* decodeSentryEnvelopeHeader(firstLine).pipe(24 const header = yield* decodeSentryEnvelopeHeader(firstLine).pipe(
54 Effect.mapError((cause) => new SentryTunnelError({ cause })),25 Effect.mapError((cause) => new SentryTunnelError({ cause })),
apps/cloud/src/start.test.ts+0−21
@@ -1,21 +0,0 @@
1import { describe, expect, it } from "@effect/vitest";
2import { Effect } from "effect";
3
4import { handleSentryTunnelRequest, MAX_SENTRY_ENVELOPE_BODY_BYTES } from "./sentry-tunnel";
5
6describe("handleSentryTunnelRequest", () => {
7 it.effect("rejects oversized Sentry tunnel envelopes before forwarding", () =>
8 Effect.gen(function* () {
9 const response = yield* handleSentryTunnelRequest(
10 new Request("https://executor.sh/api/sentry-tunnel", {
11 method: "POST",
12 body: "x".repeat(MAX_SENTRY_ENVELOPE_BODY_BYTES + 1),
13 }),
14 "https://public@example.sentry.io/123",
15 );
16
17 expect(response.status).toBe(413);
18 expect(yield* Effect.promise(() => response.text())).toBe("payload too large");
19 }),
20 );
21});
packages/core/sdk/src/executor.ts+1−1
@@ -3287,7 +3287,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = []> · 3286 unmodified lines
3287 const sourceDetectionMaxUrlLength = config.sourceDetection?.maxUrlLength ?? 2_048;3287 const sourceDetectionMaxUrlLength = config.sourceDetection?.maxUrlLength ?? 2_048;
3288 const sourceDetectionMaxDetectors = config.sourceDetection?.maxDetectors ?? 6;3288 const sourceDetectionMaxDetectors = config.sourceDetection?.maxDetectors ?? 6;
3289 const sourceDetectionMaxResults = config.sourceDetection?.maxResults ?? 4;3289 const sourceDetectionMaxResults = config.sourceDetection?.maxResults ?? 4;
3290 const sourceDetectionTimeout = config.sourceDetection?.timeout ?? "5 seconds";3290 const sourceDetectionTimeout = config.sourceDetection?.timeout ?? "60 seconds";
3291 const sourceDetectionHostedOutboundPolicy =3291 const sourceDetectionHostedOutboundPolicy =
3292 config.sourceDetection?.hostedOutboundPolicy ?? config.httpClientLayer !== undefined;3292 config.sourceDetection?.hostedOutboundPolicy ?? config.httpClientLayer !== undefined;
3293 3293
packages/core/sdk/src/hosted-http-client.test.ts+0−40
@@ -99,44 +99,4 @@ describe("hosted outbound HTTP client", () => { · 98 unmodified lines
99 expect(calls).toBe(1);99 expect(calls).toBe(1);
100 }),100 }),
101 );101 );
102
103 it.effect("rejects responses with content-length beyond the configured limit", () =>
104 Effect.gen(function* () {
105 const fakeFetch: typeof globalThis.fetch = (async () =>
106 new Response("too large", {
107 status: 200,
108 headers: { "content-length": "9" },
109 })) as typeof globalThis.fetch;
110
111 const result = yield* Effect.gen(function* () {
112 const client = yield* HttpClient.HttpClient;
113 return yield* client.execute(HttpClientRequest.get("https://public.example/spec"));
114 }).pipe(
115 Effect.provide(makeHostedHttpClientLayer({ fetch: fakeFetch, maxResponseBytes: 8 })),
116 Effect.result,
117 );
118
119 expect(Result.isFailure(result)).toBe(true);
120 }),
121 );
122
123 it.effect("rejects streamed responses that exceed the configured limit", () =>
124 Effect.gen(function* () {
125 const fakeFetch: typeof globalThis.fetch = (async () =>
126 new Response("too large", { status: 200 })) as typeof globalThis.fetch;
127
128 const result = yield* Effect.gen(function* () {
129 const client = yield* HttpClient.HttpClient;
130 const response = yield* client.execute(
131 HttpClientRequest.get("https://public.example/spec"),
132 );
133 return yield* response.text;
134 }).pipe(
135 Effect.provide(makeHostedHttpClientLayer({ fetch: fakeFetch, maxResponseBytes: 8 })),
136 Effect.result,
137 );
138
139 expect(Result.isFailure(result)).toBe(true);
140 }),
141 );
142});102});
packages/core/sdk/src/hosted-http-client.ts+2−63
@@ -12,12 +12,9 @@ export class HostedOutboundRequestBlocked extends Schema.TaggedErrorClass<Hosted · 11 unmodified lines
12export interface HostedHttpClientOptions {12export interface HostedHttpClientOptions {
13 readonly allowLocalNetwork?: boolean;13 readonly allowLocalNetwork?: boolean;
14 readonly maxRedirects?: number;14 readonly maxRedirects?: number;
15 readonly maxResponseBytes?: number;
16 readonly fetch?: typeof globalThis.fetch;15 readonly fetch?: typeof globalThis.fetch;
17}16}
18 17
19const DEFAULT_MAX_RESPONSE_BYTES = 5 * 1024 * 1024;
20
21const parseIpv4 = (hostname: string): readonly [number, number, number, number] | null => {18const parseIpv4 = (hostname: string): readonly [number, number, number, number] | null => {
22 const parts = hostname.split(".");19 const parts = hostname.split(".");
23 if (parts.length !== 4) return null;20 if (parts.length !== 4) return null;
@@ -158,69 +155,11 @@ const guardFetch = ( · 134 unmodified lines
158 current = next.toString();155 current = next.toString();
159 continue;156 continue;
160 }157 }
161 return guardResponseBody(158 return response;
162 response,
163 url,
164 options.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES,
165 );
166 }159 }
167 const url = current instanceof Request ? current.url : String(current);160 return await underlying(current, { ...init, redirect: "manual" });
168 const response = await underlying(current, { ...init, redirect: "manual" });
169 return guardResponseBody(response, url, options.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES);
170 }) as typeof globalThis.fetch;161 }) as typeof globalThis.fetch;
171 162
172const guardResponseBody = (response: Response, url: string, maxResponseBytes: number): Response => {
173 if (!Number.isFinite(maxResponseBytes) || maxResponseBytes <= 0 || !response.body) {
174 return response;
175 }
176
177 const contentLength = response.headers.get("content-length");
178 if (contentLength !== null) {
179 const parsed = Number(contentLength);
180 if (Number.isFinite(parsed) && parsed > maxResponseBytes) {
181 // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: fetch-compatible response adapter must reject oversized responses
182 throw new HostedOutboundRequestBlocked({
183 url,
184 reason: "Response body is too large",
185 });
186 }
187 }
188
189 let total = 0;
190 const reader = response.body.getReader();
191 const limitedBody = new ReadableStream<Uint8Array>({
192 async pull(controller) {
193 const next = await reader.read();
194 if (next.done) {
195 controller.close();
196 return;
197 }
198
199 total += next.value.byteLength;
200 if (total > maxResponseBytes) {
201 controller.error(
202 new HostedOutboundRequestBlocked({
203 url,
204 reason: "Response body is too large",
205 }),
206 );
207 return;
208 }
209
210 controller.enqueue(next.value);
211 },
212 cancel(reason) {
213 return reader.cancel(reason);
214 },
215 });
216
217 return new Response(limitedBody, {
218 status: response.status,
219 statusText: response.statusText,
220 headers: response.headers,
221 });
222};
223
224export const makeHostedHttpClientLayer = (163export const makeHostedHttpClientLayer = (
225 options: HostedHttpClientOptions = {},164 options: HostedHttpClientOptions = {},
226): Layer.Layer<HttpClient.HttpClient> =>165): Layer.Layer<HttpClient.HttpClient> =>
packages/plugins/openapi/src/sdk/parse.ts+2−2
@@ -20,7 +20,7 @@ class OpenApiExtractionErrorFromParse extends OpenApiExtractionError {} · 19 unmodified lines
20 * HttpClient so the caller chooses the transport via layer — in Cloudflare20 * HttpClient so the caller chooses the transport via layer — in Cloudflare
21 * Workers, `FetchHttpClient.layer` binds to the Workers-native `fetch` and21 * Workers, `FetchHttpClient.layer` binds to the Workers-native `fetch` and
22 * avoids json-schema-ref-parser's Node-polyfill http resolver, which hangs22 * avoids json-schema-ref-parser's Node-polyfill http resolver, which hangs
23 * in production. Bounded by a 20s timeout.23 * in production. Bounded by a 60s timeout.
24 */24 */
25export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* (25export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* (
26 url: string,26 url: string,
@@ -38,7 +38,7 @@ export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* ( · 11 unmodified lines
38 request = HttpClientRequest.setHeader(request, name, value);38 request = HttpClientRequest.setHeader(request, name, value);
39 }39 }
40 const response = yield* client.execute(request).pipe(40 const response = yield* client.execute(request).pipe(
41 Effect.timeout(Duration.seconds(20)),41 Effect.timeout(Duration.seconds(60)),
42 Effect.mapError(42 Effect.mapError(
43 (_cause) =>43 (_cause) =>
44 new OpenApiParseError({44 new OpenApiParseError({