[codex] Remove OpenAPI import size limits
Unified · Split
| @@ -72,7 +72,6 @@ const MCP_PATH = "/mcp"; · 71 unmodified lines | |||
| 72 | const PROTECTED_RESOURCE_METADATA_PATH = "/.well-known/oauth-protected-resource/mcp"; | 72 | const PROTECTED_RESOURCE_METADATA_PATH = "/.well-known/oauth-protected-resource/mcp"; |
| 73 | const PROTECTED_RESOURCE_METADATA_URL = `${RESOURCE_ORIGIN}${PROTECTED_RESOURCE_METADATA_PATH}`; | 73 | const PROTECTED_RESOURCE_METADATA_URL = `${RESOURCE_ORIGIN}${PROTECTED_RESOURCE_METADATA_PATH}`; |
| 74 | const RESOURCE_URL = `${RESOURCE_ORIGIN}${MCP_PATH}`; | 74 | const RESOURCE_URL = `${RESOURCE_ORIGIN}${MCP_PATH}`; |
| 75 | export const MAX_MCP_JSON_RPC_BODY_BYTES = 1_000_000; | ||
| 76 | 75 | ||
| 77 | type McpUnauthorizedReason = "missing_bearer" | "invalid_token"; | 76 | type McpUnauthorizedReason = "missing_bearer" | "invalid_token"; |
| 78 | 77 | ||
| @@ -108,18 +107,6 @@ const corsPreflight = HttpServerResponse.empty({ · 29 unmodified lines | |||
| 108 | headers: CORS_PREFLIGHT_HEADERS, | 107 | headers: CORS_PREFLIGHT_HEADERS, |
| 109 | }); | 108 | }); |
| 110 | 109 | ||
| 111 | const contentLength = (request: Request): number | null => { | ||
| 112 | const value = request.headers.get("content-length"); | ||
| 113 | if (!value) return null; | ||
| 114 | const parsed = Number(value); | ||
| 115 | return Number.isFinite(parsed) && parsed >= 0 ? parsed : null; | ||
| 116 | }; | ||
| 117 | |||
| 118 | const isBodyTooLarge = (request: Request, maxBytes: number): boolean => { | ||
| 119 | const length = contentLength(request); | ||
| 120 | return length !== null && length > maxBytes; | ||
| 121 | }; | ||
| 122 | |||
| 123 | // --------------------------------------------------------------------------- | 110 | // --------------------------------------------------------------------------- |
| 124 | // Auth | 111 | // Auth |
| 125 | // --------------------------------------------------------------------------- | 112 | // --------------------------------------------------------------------------- |
| @@ -315,11 +302,7 @@ const readJsonRpcEnvelope = (request: Request): Effect.Effect<Option.Option<Json · 189 unmodified lines | |||
| 315 | try: () => request.clone().text(), | 302 | try: () => request.clone().text(), |
| 316 | catch: () => undefined, | 303 | catch: () => undefined, |
| 317 | }).pipe( | 304 | }).pipe( |
| 318 | Effect.map((text) => | 305 | Effect.map((text) => (text ? decodeJsonRpcEnvelopeString(text) : Option.none())), |
| 319 | text && text.length <= MAX_MCP_JSON_RPC_BODY_BYTES | ||
| 320 | ? decodeJsonRpcEnvelopeString(text) | ||
| 321 | : Option.none(), | ||
| 322 | ), | ||
| 323 | Effect.catchCause(() => Effect.succeed(Option.none())), | 306 | Effect.catchCause(() => Effect.succeed(Option.none())), |
| 324 | Effect.withSpan("mcp.request.read_json_rpc"), | 307 | Effect.withSpan("mcp.request.read_json_rpc"), |
| 325 | ); | 308 | ); |
| @@ -691,23 +674,18 @@ export const mcpApp: Effect.Effect< · 365 unmodified lines | |||
| 691 | return yield* authTemporarilyUnavailable(authResult.failure); | 674 | return yield* authTemporarilyUnavailable(authResult.failure); |
| 692 | } | 675 | } |
| 693 | const authValue = authResult.success; | 676 | const authValue = authResult.success; |
| 694 | const bodyTooLarge = | ||
| 695 | request.method === "POST" && isBodyTooLarge(request, MAX_MCP_JSON_RPC_BODY_BYTES); | ||
| 696 | 677 | ||
| 697 | // Annotate before dispatch so even 401s show up with what we know. Only | 678 | // Annotate before dispatch so even 401s show up with what we know. Only |
| 698 | // POST bodies are JSON-RPC payloads worth parsing; GET (SSE) and DELETE | 679 | // POST bodies are JSON-RPC payloads worth parsing; GET (SSE) and DELETE |
| 699 | // don't carry one. | 680 | // don't carry one. |
| 700 | yield* annotateMcpRequest(request, { | 681 | yield* annotateMcpRequest(request, { |
| 701 | token: isMcpAuthorized(authValue) ? authValue.token : null, | 682 | token: isMcpAuthorized(authValue) ? authValue.token : null, |
| 702 | parseBody: request.method === "POST" && isMcpAuthorized(authValue) && !bodyTooLarge, | 683 | parseBody: request.method === "POST" && isMcpAuthorized(authValue), |
| 703 | }); | 684 | }); |
| 704 | 685 | ||
| 705 | if (isMcpUnauthorized(authValue)) { | 686 | if (isMcpUnauthorized(authValue)) { |
| 706 | return unauthorized(authValue, PROTECTED_RESOURCE_METADATA_URL); | 687 | return unauthorized(authValue, PROTECTED_RESOURCE_METADATA_URL); |
| 707 | } | 688 | } |
| 708 | if (bodyTooLarge) { | ||
| 709 | return jsonRpcError(413, -32000, "Request body too large"); | ||
| 710 | } | ||
| 711 | const token = authValue.token; | 689 | const token = authValue.token; |
| 712 | switch (request.method) { | 690 | switch (request.method) { |
| 713 | case "POST": | 691 | case "POST": |
| @@ -1,7 +1,5 @@ | |||
| 1 | import { Data, Effect, Schema } from "effect"; | 1 | import { Data, Effect, Schema } from "effect"; |
| 2 | 2 | ||
| 3 | export const MAX_SENTRY_ENVELOPE_BODY_BYTES = 200_000; | ||
| 4 | |||
| 5 | class SentryTunnelError extends Data.TaggedError("SentryTunnelError")<{ | 3 | class SentryTunnelError extends Data.TaggedError("SentryTunnelError")<{ |
| 6 | readonly cause?: unknown; | 4 | readonly cause?: unknown; |
| 7 | }> {} | 5 | }> {} |
| @@ -16,39 +14,12 @@ const decodeSentryEnvelopeHeader = Schema.decodeUnknownEffect( · 8 unmodified lines | |||
| 16 | 14 | ||
| 17 | const badSentryEnvelopeResponse = () => new Response("bad envelope", { status: 400 }); | 15 | const badSentryEnvelopeResponse = () => new Response("bad envelope", { status: 400 }); |
| 18 | 16 | ||
| 19 | const requestContentLength = (request: Request): number | null => { | 17 | export const handleSentryTunnelRequest = (request: Request, configuredDsn: string) => |
| 20 | const value = request.headers.get("content-length"); | ||
| 21 | if (!value) return null; | ||
| 22 | const parsed = Number(value); | ||
| 23 | return Number.isFinite(parsed) && parsed >= 0 ? parsed : null; | ||
| 24 | }; | ||
| 25 | |||
| 26 | const readLimitedRequestText = (request: Request, maxBytes: number) => | ||
| 27 | Effect.gen(function* () { | 18 | Effect.gen(function* () { |
| 28 | const length = requestContentLength(request); | 19 | const envelope = yield* Effect.tryPromise({ |
| 29 | if (length !== null && length > maxBytes) { | ||
| 30 | return yield* new SentryTunnelError({ cause: "payload_too_large" }); | ||
| 31 | } | ||
| 32 | const text = yield* Effect.tryPromise({ | ||
| 33 | try: () => request.text(), | 20 | try: () => request.text(), |
| 34 | catch: (cause) => new SentryTunnelError({ cause }), | 21 | catch: (cause) => new SentryTunnelError({ cause }), |
| 35 | }); | 22 | }); |
| 36 | if (text.length > maxBytes) { | ||
| 37 | return yield* new SentryTunnelError({ cause: "payload_too_large" }); | ||
| 38 | } | ||
| 39 | return text; | ||
| 40 | }); | ||
| 41 | |||
| 42 | export const handleSentryTunnelRequest = (request: Request, configuredDsn: string) => | ||
| 43 | Effect.gen(function* () { | ||
| 44 | const envelope = yield* readLimitedRequestText(request, MAX_SENTRY_ENVELOPE_BODY_BYTES).pipe( | ||
| 45 | Effect.catchTag("SentryTunnelError", (error) => | ||
| 46 | error.cause === "payload_too_large" | ||
| 47 | ? Effect.succeed(new Response("payload too large", { status: 413 })) | ||
| 48 | : Effect.fail(error), | ||
| 49 | ), | ||
| 50 | ); | ||
| 51 | if (envelope instanceof Response) return envelope; | ||
| 52 | const firstLine = envelope.slice(0, envelope.indexOf("\n")); | 23 | const firstLine = envelope.slice(0, envelope.indexOf("\n")); |
| 53 | const header = yield* decodeSentryEnvelopeHeader(firstLine).pipe( | 24 | const header = yield* decodeSentryEnvelopeHeader(firstLine).pipe( |
| 54 | Effect.mapError((cause) => new SentryTunnelError({ cause })), | 25 | Effect.mapError((cause) => new SentryTunnelError({ cause })), |
| @@ -1,21 +0,0 @@ | |||
| 1 | import { describe, expect, it } from "@effect/vitest"; | ||
| 2 | import { Effect } from "effect"; | ||
| 3 | |||
| 4 | import { handleSentryTunnelRequest, MAX_SENTRY_ENVELOPE_BODY_BYTES } from "./sentry-tunnel"; | ||
| 5 | |||
| 6 | describe("handleSentryTunnelRequest", () => { | ||
| 7 | it.effect("rejects oversized Sentry tunnel envelopes before forwarding", () => | ||
| 8 | Effect.gen(function* () { | ||
| 9 | const response = yield* handleSentryTunnelRequest( | ||
| 10 | new Request("https://executor.sh/api/sentry-tunnel", { | ||
| 11 | method: "POST", | ||
| 12 | body: "x".repeat(MAX_SENTRY_ENVELOPE_BODY_BYTES + 1), | ||
| 13 | }), | ||
| 14 | "https://public@example.sentry.io/123", | ||
| 15 | ); | ||
| 16 | |||
| 17 | expect(response.status).toBe(413); | ||
| 18 | expect(yield* Effect.promise(() => response.text())).toBe("payload too large"); | ||
| 19 | }), | ||
| 20 | ); | ||
| 21 | }); | ||
| @@ -3287,7 +3287,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = []> · 3286 unmodified lines | |||
| 3287 | const sourceDetectionMaxUrlLength = config.sourceDetection?.maxUrlLength ?? 2_048; | 3287 | const sourceDetectionMaxUrlLength = config.sourceDetection?.maxUrlLength ?? 2_048; |
| 3288 | const sourceDetectionMaxDetectors = config.sourceDetection?.maxDetectors ?? 6; | 3288 | const sourceDetectionMaxDetectors = config.sourceDetection?.maxDetectors ?? 6; |
| 3289 | const sourceDetectionMaxResults = config.sourceDetection?.maxResults ?? 4; | 3289 | const sourceDetectionMaxResults = config.sourceDetection?.maxResults ?? 4; |
| 3290 | const sourceDetectionTimeout = config.sourceDetection?.timeout ?? "5 seconds"; | 3290 | const sourceDetectionTimeout = config.sourceDetection?.timeout ?? "60 seconds"; |
| 3291 | const sourceDetectionHostedOutboundPolicy = | 3291 | const sourceDetectionHostedOutboundPolicy = |
| 3292 | config.sourceDetection?.hostedOutboundPolicy ?? config.httpClientLayer !== undefined; | 3292 | config.sourceDetection?.hostedOutboundPolicy ?? config.httpClientLayer !== undefined; |
| 3293 | 3293 | ||
| @@ -99,44 +99,4 @@ describe("hosted outbound HTTP client", () => { · 98 unmodified lines | |||
| 99 | expect(calls).toBe(1); | 99 | expect(calls).toBe(1); |
| 100 | }), | 100 | }), |
| 101 | ); | 101 | ); |
| 102 | |||
| 103 | it.effect("rejects responses with content-length beyond the configured limit", () => | ||
| 104 | Effect.gen(function* () { | ||
| 105 | const fakeFetch: typeof globalThis.fetch = (async () => | ||
| 106 | new Response("too large", { | ||
| 107 | status: 200, | ||
| 108 | headers: { "content-length": "9" }, | ||
| 109 | })) as typeof globalThis.fetch; | ||
| 110 | |||
| 111 | const result = yield* Effect.gen(function* () { | ||
| 112 | const client = yield* HttpClient.HttpClient; | ||
| 113 | return yield* client.execute(HttpClientRequest.get("https://public.example/spec")); | ||
| 114 | }).pipe( | ||
| 115 | Effect.provide(makeHostedHttpClientLayer({ fetch: fakeFetch, maxResponseBytes: 8 })), | ||
| 116 | Effect.result, | ||
| 117 | ); | ||
| 118 | |||
| 119 | expect(Result.isFailure(result)).toBe(true); | ||
| 120 | }), | ||
| 121 | ); | ||
| 122 | |||
| 123 | it.effect("rejects streamed responses that exceed the configured limit", () => | ||
| 124 | Effect.gen(function* () { | ||
| 125 | const fakeFetch: typeof globalThis.fetch = (async () => | ||
| 126 | new Response("too large", { status: 200 })) as typeof globalThis.fetch; | ||
| 127 | |||
| 128 | const result = yield* Effect.gen(function* () { | ||
| 129 | const client = yield* HttpClient.HttpClient; | ||
| 130 | const response = yield* client.execute( | ||
| 131 | HttpClientRequest.get("https://public.example/spec"), | ||
| 132 | ); | ||
| 133 | return yield* response.text; | ||
| 134 | }).pipe( | ||
| 135 | Effect.provide(makeHostedHttpClientLayer({ fetch: fakeFetch, maxResponseBytes: 8 })), | ||
| 136 | Effect.result, | ||
| 137 | ); | ||
| 138 | |||
| 139 | expect(Result.isFailure(result)).toBe(true); | ||
| 140 | }), | ||
| 141 | ); | ||
| 142 | }); | 102 | }); |
| @@ -12,12 +12,9 @@ export class HostedOutboundRequestBlocked extends Schema.TaggedErrorClass<Hosted · 11 unmodified lines | |||
| 12 | export interface HostedHttpClientOptions { | 12 | export interface HostedHttpClientOptions { |
| 13 | readonly allowLocalNetwork?: boolean; | 13 | readonly allowLocalNetwork?: boolean; |
| 14 | readonly maxRedirects?: number; | 14 | readonly maxRedirects?: number; |
| 15 | readonly maxResponseBytes?: number; | ||
| 16 | readonly fetch?: typeof globalThis.fetch; | 15 | readonly fetch?: typeof globalThis.fetch; |
| 17 | } | 16 | } |
| 18 | 17 | ||
| 19 | const DEFAULT_MAX_RESPONSE_BYTES = 5 * 1024 * 1024; | ||
| 20 | |||
| 21 | const parseIpv4 = (hostname: string): readonly [number, number, number, number] | null => { | 18 | const parseIpv4 = (hostname: string): readonly [number, number, number, number] | null => { |
| 22 | const parts = hostname.split("."); | 19 | const parts = hostname.split("."); |
| 23 | if (parts.length !== 4) return null; | 20 | if (parts.length !== 4) return null; |
| @@ -158,69 +155,11 @@ const guardFetch = ( · 134 unmodified lines | |||
| 158 | current = next.toString(); | 155 | current = next.toString(); |
| 159 | continue; | 156 | continue; |
| 160 | } | 157 | } |
| 161 | return guardResponseBody( | 158 | return response; |
| 162 | response, | ||
| 163 | url, | ||
| 164 | options.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES, | ||
| 165 | ); | ||
| 166 | } | 159 | } |
| 167 | const url = current instanceof Request ? current.url : String(current); | 160 | return await underlying(current, { ...init, redirect: "manual" }); |
| 168 | const response = await underlying(current, { ...init, redirect: "manual" }); | ||
| 169 | return guardResponseBody(response, url, options.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES); | ||
| 170 | }) as typeof globalThis.fetch; | 161 | }) as typeof globalThis.fetch; |
| 171 | 162 | ||
| 172 | const guardResponseBody = (response: Response, url: string, maxResponseBytes: number): Response => { | ||
| 173 | if (!Number.isFinite(maxResponseBytes) || maxResponseBytes <= 0 || !response.body) { | ||
| 174 | return response; | ||
| 175 | } | ||
| 176 | |||
| 177 | const contentLength = response.headers.get("content-length"); | ||
| 178 | if (contentLength !== null) { | ||
| 179 | const parsed = Number(contentLength); | ||
| 180 | if (Number.isFinite(parsed) && parsed > maxResponseBytes) { | ||
| 181 | // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: fetch-compatible response adapter must reject oversized responses | ||
| 182 | throw new HostedOutboundRequestBlocked({ | ||
| 183 | url, | ||
| 184 | reason: "Response body is too large", | ||
| 185 | }); | ||
| 186 | } | ||
| 187 | } | ||
| 188 | |||
| 189 | let total = 0; | ||
| 190 | const reader = response.body.getReader(); | ||
| 191 | const limitedBody = new ReadableStream<Uint8Array>({ | ||
| 192 | async pull(controller) { | ||
| 193 | const next = await reader.read(); | ||
| 194 | if (next.done) { | ||
| 195 | controller.close(); | ||
| 196 | return; | ||
| 197 | } | ||
| 198 | |||
| 199 | total += next.value.byteLength; | ||
| 200 | if (total > maxResponseBytes) { | ||
| 201 | controller.error( | ||
| 202 | new HostedOutboundRequestBlocked({ | ||
| 203 | url, | ||
| 204 | reason: "Response body is too large", | ||
| 205 | }), | ||
| 206 | ); | ||
| 207 | return; | ||
| 208 | } | ||
| 209 | |||
| 210 | controller.enqueue(next.value); | ||
| 211 | }, | ||
| 212 | cancel(reason) { | ||
| 213 | return reader.cancel(reason); | ||
| 214 | }, | ||
| 215 | }); | ||
| 216 | |||
| 217 | return new Response(limitedBody, { | ||
| 218 | status: response.status, | ||
| 219 | statusText: response.statusText, | ||
| 220 | headers: response.headers, | ||
| 221 | }); | ||
| 222 | }; | ||
| 223 | |||
| 224 | export const makeHostedHttpClientLayer = ( | 163 | export const makeHostedHttpClientLayer = ( |
| 225 | options: HostedHttpClientOptions = {}, | 164 | options: HostedHttpClientOptions = {}, |
| 226 | ): Layer.Layer<HttpClient.HttpClient> => | 165 | ): Layer.Layer<HttpClient.HttpClient> => |
| @@ -20,7 +20,7 @@ class OpenApiExtractionErrorFromParse extends OpenApiExtractionError {} · 19 unmodified lines | |||
| 20 | * HttpClient so the caller chooses the transport via layer — in Cloudflare | 20 | * HttpClient so the caller chooses the transport via layer — in Cloudflare |
| 21 | * Workers, `FetchHttpClient.layer` binds to the Workers-native `fetch` and | 21 | * Workers, `FetchHttpClient.layer` binds to the Workers-native `fetch` and |
| 22 | * avoids json-schema-ref-parser's Node-polyfill http resolver, which hangs | 22 | * avoids json-schema-ref-parser's Node-polyfill http resolver, which hangs |
| 23 | * in production. Bounded by a 20s timeout. | 23 | * in production. Bounded by a 60s timeout. |
| 24 | */ | 24 | */ |
| 25 | export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* ( | 25 | export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* ( |
| 26 | url: string, | 26 | url: string, |
| @@ -38,7 +38,7 @@ export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* ( · 11 unmodified lines | |||
| 38 | request = HttpClientRequest.setHeader(request, name, value); | 38 | request = HttpClientRequest.setHeader(request, name, value); |
| 39 | } | 39 | } |
| 40 | const response = yield* client.execute(request).pipe( | 40 | const response = yield* client.execute(request).pipe( |
| 41 | Effect.timeout(Duration.seconds(20)), | 41 | Effect.timeout(Duration.seconds(60)), |
| 42 | Effect.mapError( | 42 | Effect.mapError( |
| 43 | (_cause) => | 43 | (_cause) => |
| 44 | new OpenApiParseError({ | 44 | new OpenApiParseError({ |