History

Merge pull request #730 from RhysSullivan/codex/remove-openapi-import-limits

[codex] Remove OpenAPI import size limits

Rhys Sullivan committed 6 months ago

Commit c4891d22 parents a21e52c b1066307 files · +9 −182Browse files

Unified · Split

7 files
  1. apps/cloud/src/mcp.ts +2 −24
  2. apps/cloud/src/sentry-tunnel.ts +2 −31
  3. apps/cloud/src/start.test.ts +0 −21
  4. packages/core/sdk/src/executor.ts +1 −1
  5. packages/core/sdk/src/hosted-http-client.test.ts +0 −40
  6. packages/core/sdk/src/hosted-http-client.ts +2 −63
  7. packages/plugins/openapi/src/sdk/parse.ts +2 −2
apps/cloud/src/mcp.ts+2−24
@@ -72,7 +72,6 @@ const MCP_PATH = "/mcp"; · 71 unmodified lines
7272 const PROTECTED_RESOURCE_METADATA_PATH = "/.well-known/oauth-protected-resource/mcp";
7373 const PROTECTED_RESOURCE_METADATA_URL = `${RESOURCE_ORIGIN}${PROTECTED_RESOURCE_METADATA_PATH}`;
7474 const RESOURCE_URL = `${RESOURCE_ORIGIN}${MCP_PATH}`;
75−export const MAX_MCP_JSON_RPC_BODY_BYTES = 1_000_000;
7675
7776 type McpUnauthorizedReason = "missing_bearer" | "invalid_token";
7877
@@ -108,18 +107,6 @@ const corsPreflight = HttpServerResponse.empty({ · 29 unmodified lines
108107 headers: CORS_PREFLIGHT_HEADERS,
109108 });
110109
111−const contentLength = (request: Request): number | null => {
112− const value = request.headers.get("content-length");
113− if (!value) return null;
114− const parsed = Number(value);
115− return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
116−};
117−
118−const isBodyTooLarge = (request: Request, maxBytes: number): boolean => {
119− const length = contentLength(request);
120− return length !== null && length > maxBytes;
121−};
122−
123110 // ---------------------------------------------------------------------------
124111 // Auth
125112 // ---------------------------------------------------------------------------
@@ -315,11 +302,7 @@ const readJsonRpcEnvelope = (request: Request): Effect.Effect<Option.Option<Json · 189 unmodified lines
315302 try: () => request.clone().text(),
316303 catch: () => undefined,
317304 }).pipe(
318− Effect.map((text) =>
319− text && text.length <= MAX_MCP_JSON_RPC_BODY_BYTES
320− ? decodeJsonRpcEnvelopeString(text)
321− : Option.none(),
322− ),
305+ Effect.map((text) => (text ? decodeJsonRpcEnvelopeString(text) : Option.none())),
323306 Effect.catchCause(() => Effect.succeed(Option.none())),
324307 Effect.withSpan("mcp.request.read_json_rpc"),
325308 );
@@ -691,23 +674,18 @@ export const mcpApp: Effect.Effect< · 365 unmodified lines
691674 return yield* authTemporarilyUnavailable(authResult.failure);
692675 }
693676 const authValue = authResult.success;
694− const bodyTooLarge =
695− request.method === "POST" && isBodyTooLarge(request, MAX_MCP_JSON_RPC_BODY_BYTES);
696677
697678 // Annotate before dispatch so even 401s show up with what we know. Only
698679 // POST bodies are JSON-RPC payloads worth parsing; GET (SSE) and DELETE
699680 // don't carry one.
700681 yield* annotateMcpRequest(request, {
701682 token: isMcpAuthorized(authValue) ? authValue.token : null,
702− parseBody: request.method === "POST" && isMcpAuthorized(authValue) && !bodyTooLarge,
683+ parseBody: request.method === "POST" && isMcpAuthorized(authValue),
703684 });
704685
705686 if (isMcpUnauthorized(authValue)) {
706687 return unauthorized(authValue, PROTECTED_RESOURCE_METADATA_URL);
707688 }
708− if (bodyTooLarge) {
709− return jsonRpcError(413, -32000, "Request body too large");
710− }
711689 const token = authValue.token;
712690 switch (request.method) {
713691 case "POST":
apps/cloud/src/sentry-tunnel.ts+2−31
@@ -1,7 +1,5 @@
11 import { Data, Effect, Schema } from "effect";
22
3−export const MAX_SENTRY_ENVELOPE_BODY_BYTES = 200_000;
4−
53 class SentryTunnelError extends Data.TaggedError("SentryTunnelError")<{
64 readonly cause?: unknown;
75 }> {}
@@ -16,39 +14,12 @@ const decodeSentryEnvelopeHeader = Schema.decodeUnknownEffect( · 8 unmodified lines
1614
1715 const badSentryEnvelopeResponse = () => new Response("bad envelope", { status: 400 });
1816
19−const requestContentLength = (request: Request): number | null => {
20− const value = request.headers.get("content-length");
21− if (!value) return null;
22− const parsed = Number(value);
23− return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
24−};
25−
26−const readLimitedRequestText = (request: Request, maxBytes: number) =>
17+export const handleSentryTunnelRequest = (request: Request, configuredDsn: string) =>
2718 Effect.gen(function* () {
28− const length = requestContentLength(request);
29− if (length !== null && length > maxBytes) {
30− return yield* new SentryTunnelError({ cause: "payload_too_large" });
31− }
32− const text = yield* Effect.tryPromise({
19+ const envelope = yield* Effect.tryPromise({
3320 try: () => request.text(),
3421 catch: (cause) => new SentryTunnelError({ cause }),
3522 });
36− if (text.length > maxBytes) {
37− return yield* new SentryTunnelError({ cause: "payload_too_large" });
38− }
39− return text;
40− });
41−
42−export const handleSentryTunnelRequest = (request: Request, configuredDsn: string) =>
43− Effect.gen(function* () {
44− const envelope = yield* readLimitedRequestText(request, MAX_SENTRY_ENVELOPE_BODY_BYTES).pipe(
45− Effect.catchTag("SentryTunnelError", (error) =>
46− error.cause === "payload_too_large"
47− ? Effect.succeed(new Response("payload too large", { status: 413 }))
48− : Effect.fail(error),
49− ),
50− );
51− if (envelope instanceof Response) return envelope;
5223 const firstLine = envelope.slice(0, envelope.indexOf("\n"));
5324 const header = yield* decodeSentryEnvelopeHeader(firstLine).pipe(
5425 Effect.mapError((cause) => new SentryTunnelError({ cause })),
apps/cloud/src/start.test.ts+0−21
@@ -1,21 +0,0 @@
1−import { describe, expect, it } from "@effect/vitest";
2−import { Effect } from "effect";
3−
4−import { handleSentryTunnelRequest, MAX_SENTRY_ENVELOPE_BODY_BYTES } from "./sentry-tunnel";
5−
6−describe("handleSentryTunnelRequest", () => {
7− it.effect("rejects oversized Sentry tunnel envelopes before forwarding", () =>
8− Effect.gen(function* () {
9− const response = yield* handleSentryTunnelRequest(
10− new Request("https://executor.sh/api/sentry-tunnel", {
11− method: "POST",
12− body: "x".repeat(MAX_SENTRY_ENVELOPE_BODY_BYTES + 1),
13− }),
14− "https://public@example.sentry.io/123",
15− );
16−
17− expect(response.status).toBe(413);
18− expect(yield* Effect.promise(() => response.text())).toBe("payload too large");
19− }),
20− );
21−});
packages/core/sdk/src/executor.ts+1−1
@@ -3287,7 +3287,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = []> · 3286 unmodified lines
32873287 const sourceDetectionMaxUrlLength = config.sourceDetection?.maxUrlLength ?? 2_048;
32883288 const sourceDetectionMaxDetectors = config.sourceDetection?.maxDetectors ?? 6;
32893289 const sourceDetectionMaxResults = config.sourceDetection?.maxResults ?? 4;
3290− const sourceDetectionTimeout = config.sourceDetection?.timeout ?? "5 seconds";
3290+ const sourceDetectionTimeout = config.sourceDetection?.timeout ?? "60 seconds";
32913291 const sourceDetectionHostedOutboundPolicy =
32923292 config.sourceDetection?.hostedOutboundPolicy ?? config.httpClientLayer !== undefined;
32933293
packages/core/sdk/src/hosted-http-client.test.ts+0−40
@@ -99,44 +99,4 @@ describe("hosted outbound HTTP client", () => { · 98 unmodified lines
9999 expect(calls).toBe(1);
100100 }),
101101 );
102−
103− it.effect("rejects responses with content-length beyond the configured limit", () =>
104− Effect.gen(function* () {
105− const fakeFetch: typeof globalThis.fetch = (async () =>
106− new Response("too large", {
107− status: 200,
108− headers: { "content-length": "9" },
109− })) as typeof globalThis.fetch;
110−
111− const result = yield* Effect.gen(function* () {
112− const client = yield* HttpClient.HttpClient;
113− return yield* client.execute(HttpClientRequest.get("https://public.example/spec"));
114− }).pipe(
115− Effect.provide(makeHostedHttpClientLayer({ fetch: fakeFetch, maxResponseBytes: 8 })),
116− Effect.result,
117− );
118−
119− expect(Result.isFailure(result)).toBe(true);
120− }),
121− );
122−
123− it.effect("rejects streamed responses that exceed the configured limit", () =>
124− Effect.gen(function* () {
125− const fakeFetch: typeof globalThis.fetch = (async () =>
126− new Response("too large", { status: 200 })) as typeof globalThis.fetch;
127−
128− const result = yield* Effect.gen(function* () {
129− const client = yield* HttpClient.HttpClient;
130− const response = yield* client.execute(
131− HttpClientRequest.get("https://public.example/spec"),
132− );
133− return yield* response.text;
134− }).pipe(
135− Effect.provide(makeHostedHttpClientLayer({ fetch: fakeFetch, maxResponseBytes: 8 })),
136− Effect.result,
137− );
138−
139− expect(Result.isFailure(result)).toBe(true);
140− }),
141− );
142102 });
packages/core/sdk/src/hosted-http-client.ts+2−63
@@ -12,12 +12,9 @@ export class HostedOutboundRequestBlocked extends Schema.TaggedErrorClass<Hosted · 11 unmodified lines
1212 export interface HostedHttpClientOptions {
1313 readonly allowLocalNetwork?: boolean;
1414 readonly maxRedirects?: number;
15− readonly maxResponseBytes?: number;
1615 readonly fetch?: typeof globalThis.fetch;
1716 }
1817
19−const DEFAULT_MAX_RESPONSE_BYTES = 5 * 1024 * 1024;
20−
2118 const parseIpv4 = (hostname: string): readonly [number, number, number, number] | null => {
2219 const parts = hostname.split(".");
2320 if (parts.length !== 4) return null;
@@ -158,69 +155,11 @@ const guardFetch = ( · 134 unmodified lines
158155 current = next.toString();
159156 continue;
160157 }
161− return guardResponseBody(
162− response,
163− url,
164− options.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES,
165− );
158+ return response;
166159 }
167− const url = current instanceof Request ? current.url : String(current);
168− const response = await underlying(current, { ...init, redirect: "manual" });
169− return guardResponseBody(response, url, options.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES);
160+ return await underlying(current, { ...init, redirect: "manual" });
170161 }) as typeof globalThis.fetch;
171162
172−const guardResponseBody = (response: Response, url: string, maxResponseBytes: number): Response => {
173− if (!Number.isFinite(maxResponseBytes) || maxResponseBytes <= 0 || !response.body) {
174− return response;
175− }
176−
177− const contentLength = response.headers.get("content-length");
178− if (contentLength !== null) {
179− const parsed = Number(contentLength);
180− if (Number.isFinite(parsed) && parsed > maxResponseBytes) {
181− // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: fetch-compatible response adapter must reject oversized responses
182− throw new HostedOutboundRequestBlocked({
183− url,
184− reason: "Response body is too large",
185− });
186− }
187− }
188−
189− let total = 0;
190− const reader = response.body.getReader();
191− const limitedBody = new ReadableStream<Uint8Array>({
192− async pull(controller) {
193− const next = await reader.read();
194− if (next.done) {
195− controller.close();
196− return;
197− }
198−
199− total += next.value.byteLength;
200− if (total > maxResponseBytes) {
201− controller.error(
202− new HostedOutboundRequestBlocked({
203− url,
204− reason: "Response body is too large",
205− }),
206− );
207− return;
208− }
209−
210− controller.enqueue(next.value);
211− },
212− cancel(reason) {
213− return reader.cancel(reason);
214− },
215− });
216−
217− return new Response(limitedBody, {
218− status: response.status,
219− statusText: response.statusText,
220− headers: response.headers,
221− });
222−};
223−
224163 export const makeHostedHttpClientLayer = (
225164 options: HostedHttpClientOptions = {},
226165 ): Layer.Layer<HttpClient.HttpClient> =>
packages/plugins/openapi/src/sdk/parse.ts+2−2
@@ -20,7 +20,7 @@ class OpenApiExtractionErrorFromParse extends OpenApiExtractionError {} · 19 unmodified lines
2020 * HttpClient so the caller chooses the transport via layer — in Cloudflare
2121 * Workers, `FetchHttpClient.layer` binds to the Workers-native `fetch` and
2222 * avoids json-schema-ref-parser's Node-polyfill http resolver, which hangs
23− * in production. Bounded by a 20s timeout.
23+ * in production. Bounded by a 60s timeout.
2424 */
2525 export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* (
2626 url: string,
@@ -38,7 +38,7 @@ export const fetchSpecText = Effect.fn("OpenApi.fetchSpecText")(function* ( · 11 unmodified lines
3838 request = HttpClientRequest.setHeader(request, name, value);
3939 }
4040 const response = yield* client.execute(request).pipe(
41− Effect.timeout(Duration.seconds(20)),
41+ Effect.timeout(Duration.seconds(60)),
4242 Effect.mapError(
4343 (_cause) =>
4444 new OpenApiParseError({