History

feat(v2): replace OpenAPI extraction with Rust WASM

Move OpenAPI extraction to a single Rust-backed WebAssembly path so large specs process much faster while keeping one maintainable runtime flow across environments.

Rhys Sullivan committed 8 months ago

Commit 7213d701 parent 322f1b916 files · +1644 −628Browse files

Showing the changes to tools (7 of 16 files). Show all

Unified · Split

7 files
  1. tools/openapi-extractor-rs/.gitignore +1 −0
  2. tools/openapi-extractor-rs/Cargo.lock +445 −0
  3. tools/openapi-extractor-rs/Cargo.toml +17 −0
  4. tools/openapi-extractor-rs/README.md +33 −0
  5. tools/openapi-extractor-rs/build-wasm.sh +20 −0
  6. tools/openapi-extractor-rs/src/lib.rs +690 −0
  7. tools/openapi-extractor-rs/src/main.rs +58 −0
tools/openapi-extractor-rs/.gitignore+1−0
@@ -0,0 +1 @@
1+target/
tools/openapi-extractor-rs/Cargo.lock+445−0
@@ -0,0 +1,445 @@
1+# This file is automatically @generated by Cargo.
2+# It is not intended for manual editing.
3+version = 4
4+
5+[[package]]
6+name = "anstream"
7+version = "0.6.21"
8+source = "registry+https://github.com/rust-lang/crates.io-index"
9+checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
10+dependencies = [
11+ "anstyle",
12+ "anstyle-parse",
13+ "anstyle-query",
14+ "anstyle-wincon",
15+ "colorchoice",
16+ "is_terminal_polyfill",
17+ "utf8parse",
18+]
19+
20+[[package]]
21+name = "anstyle"
22+version = "1.0.13"
23+source = "registry+https://github.com/rust-lang/crates.io-index"
24+checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
25+
26+[[package]]
27+name = "anstyle-parse"
28+version = "0.2.7"
29+source = "registry+https://github.com/rust-lang/crates.io-index"
30+checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
31+dependencies = [
32+ "utf8parse",
33+]
34+
35+[[package]]
36+name = "anstyle-query"
37+version = "1.1.5"
38+source = "registry+https://github.com/rust-lang/crates.io-index"
39+checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
40+dependencies = [
41+ "windows-sys",
42+]
43+
44+[[package]]
45+name = "anstyle-wincon"
46+version = "3.0.11"
47+source = "registry+https://github.com/rust-lang/crates.io-index"
48+checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
49+dependencies = [
50+ "anstyle",
51+ "once_cell_polyfill",
52+ "windows-sys",
53+]
54+
55+[[package]]
56+name = "block-buffer"
57+version = "0.10.4"
58+source = "registry+https://github.com/rust-lang/crates.io-index"
59+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
60+dependencies = [
61+ "generic-array",
62+]
63+
64+[[package]]
65+name = "bumpalo"
66+version = "3.20.2"
67+source = "registry+https://github.com/rust-lang/crates.io-index"
68+checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
69+
70+[[package]]
71+name = "cfg-if"
72+version = "1.0.4"
73+source = "registry+https://github.com/rust-lang/crates.io-index"
74+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
75+
76+[[package]]
77+name = "clap"
78+version = "4.5.60"
79+source = "registry+https://github.com/rust-lang/crates.io-index"
80+checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a"
81+dependencies = [
82+ "clap_builder",
83+ "clap_derive",
84+]
85+
86+[[package]]
87+name = "clap_builder"
88+version = "4.5.60"
89+source = "registry+https://github.com/rust-lang/crates.io-index"
90+checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876"
91+dependencies = [
92+ "anstream",
93+ "anstyle",
94+ "clap_lex",
95+ "strsim",
96+]
97+
98+[[package]]
99+name = "clap_derive"
100+version = "4.5.55"
101+source = "registry+https://github.com/rust-lang/crates.io-index"
102+checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5"
103+dependencies = [
104+ "heck",
105+ "proc-macro2",
106+ "quote",
107+ "syn",
108+]
109+
110+[[package]]
111+name = "clap_lex"
112+version = "1.0.0"
113+source = "registry+https://github.com/rust-lang/crates.io-index"
114+checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831"
115+
116+[[package]]
117+name = "colorchoice"
118+version = "1.0.4"
119+source = "registry+https://github.com/rust-lang/crates.io-index"
120+checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
121+
122+[[package]]
123+name = "cpufeatures"
124+version = "0.2.17"
125+source = "registry+https://github.com/rust-lang/crates.io-index"
126+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
127+dependencies = [
128+ "libc",
129+]
130+
131+[[package]]
132+name = "crypto-common"
133+version = "0.1.7"
134+source = "registry+https://github.com/rust-lang/crates.io-index"
135+checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
136+dependencies = [
137+ "generic-array",
138+ "typenum",
139+]
140+
141+[[package]]
142+name = "digest"
143+version = "0.10.7"
144+source = "registry+https://github.com/rust-lang/crates.io-index"
145+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
146+dependencies = [
147+ "block-buffer",
148+ "crypto-common",
149+]
150+
151+[[package]]
152+name = "equivalent"
153+version = "1.0.2"
154+source = "registry+https://github.com/rust-lang/crates.io-index"
155+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
156+
157+[[package]]
158+name = "generic-array"
159+version = "0.14.7"
160+source = "registry+https://github.com/rust-lang/crates.io-index"
161+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
162+dependencies = [
163+ "typenum",
164+ "version_check",
165+]
166+
167+[[package]]
168+name = "hashbrown"
169+version = "0.16.1"
170+source = "registry+https://github.com/rust-lang/crates.io-index"
171+checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
172+
173+[[package]]
174+name = "heck"
175+version = "0.5.0"
176+source = "registry+https://github.com/rust-lang/crates.io-index"
177+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
178+
179+[[package]]
180+name = "indexmap"
181+version = "2.13.0"
182+source = "registry+https://github.com/rust-lang/crates.io-index"
183+checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
184+dependencies = [
185+ "equivalent",
186+ "hashbrown",
187+]
188+
189+[[package]]
190+name = "is_terminal_polyfill"
191+version = "1.70.2"
192+source = "registry+https://github.com/rust-lang/crates.io-index"
193+checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
194+
195+[[package]]
196+name = "itoa"
197+version = "1.0.17"
198+source = "registry+https://github.com/rust-lang/crates.io-index"
199+checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
200+
201+[[package]]
202+name = "libc"
203+version = "0.2.182"
204+source = "registry+https://github.com/rust-lang/crates.io-index"
205+checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
206+
207+[[package]]
208+name = "memchr"
209+version = "2.8.0"
210+source = "registry+https://github.com/rust-lang/crates.io-index"
211+checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
212+
213+[[package]]
214+name = "once_cell"
215+version = "1.21.3"
216+source = "registry+https://github.com/rust-lang/crates.io-index"
217+checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
218+
219+[[package]]
220+name = "once_cell_polyfill"
221+version = "1.70.2"
222+source = "registry+https://github.com/rust-lang/crates.io-index"
223+checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
224+
225+[[package]]
226+name = "openapi-extractor-rs"
227+version = "0.1.0"
228+dependencies = [
229+ "clap",
230+ "serde",
231+ "serde_json",
232+ "serde_yaml",
233+ "sha2",
234+ "wasm-bindgen",
235+]
236+
237+[[package]]
238+name = "proc-macro2"
239+version = "1.0.106"
240+source = "registry+https://github.com/rust-lang/crates.io-index"
241+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
242+dependencies = [
243+ "unicode-ident",
244+]
245+
246+[[package]]
247+name = "quote"
248+version = "1.0.44"
249+source = "registry+https://github.com/rust-lang/crates.io-index"
250+checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
251+dependencies = [
252+ "proc-macro2",
253+]
254+
255+[[package]]
256+name = "rustversion"
257+version = "1.0.22"
258+source = "registry+https://github.com/rust-lang/crates.io-index"
259+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
260+
261+[[package]]
262+name = "ryu"
263+version = "1.0.23"
264+source = "registry+https://github.com/rust-lang/crates.io-index"
265+checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
266+
267+[[package]]
268+name = "serde"
269+version = "1.0.228"
270+source = "registry+https://github.com/rust-lang/crates.io-index"
271+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
272+dependencies = [
273+ "serde_core",
274+ "serde_derive",
275+]
276+
277+[[package]]
278+name = "serde_core"
279+version = "1.0.228"
280+source = "registry+https://github.com/rust-lang/crates.io-index"
281+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
282+dependencies = [
283+ "serde_derive",
284+]
285+
286+[[package]]
287+name = "serde_derive"
288+version = "1.0.228"
289+source = "registry+https://github.com/rust-lang/crates.io-index"
290+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
291+dependencies = [
292+ "proc-macro2",
293+ "quote",
294+ "syn",
295+]
296+
297+[[package]]
298+name = "serde_json"
299+version = "1.0.149"
300+source = "registry+https://github.com/rust-lang/crates.io-index"
301+checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
302+dependencies = [
303+ "itoa",
304+ "memchr",
305+ "serde",
306+ "serde_core",
307+ "zmij",
308+]
309+
310+[[package]]
311+name = "serde_yaml"
312+version = "0.9.34+deprecated"
313+source = "registry+https://github.com/rust-lang/crates.io-index"
314+checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
315+dependencies = [
316+ "indexmap",
317+ "itoa",
318+ "ryu",
319+ "serde",
320+ "unsafe-libyaml",
321+]
322+
323+[[package]]
324+name = "sha2"
325+version = "0.10.9"
326+source = "registry+https://github.com/rust-lang/crates.io-index"
327+checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
328+dependencies = [
329+ "cfg-if",
330+ "cpufeatures",
331+ "digest",
332+]
333+
334+[[package]]
335+name = "strsim"
336+version = "0.11.1"
337+source = "registry+https://github.com/rust-lang/crates.io-index"
338+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
339+
340+[[package]]
341+name = "syn"
342+version = "2.0.117"
343+source = "registry+https://github.com/rust-lang/crates.io-index"
344+checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
345+dependencies = [
346+ "proc-macro2",
347+ "quote",
348+ "unicode-ident",
349+]
350+
351+[[package]]
352+name = "typenum"
353+version = "1.19.0"
354+source = "registry+https://github.com/rust-lang/crates.io-index"
355+checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
356+
357+[[package]]
358+name = "unicode-ident"
359+version = "1.0.24"
360+source = "registry+https://github.com/rust-lang/crates.io-index"
361+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
362+
363+[[package]]
364+name = "unsafe-libyaml"
365+version = "0.2.11"
366+source = "registry+https://github.com/rust-lang/crates.io-index"
367+checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
368+
369+[[package]]
370+name = "utf8parse"
371+version = "0.2.2"
372+source = "registry+https://github.com/rust-lang/crates.io-index"
373+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
374+
375+[[package]]
376+name = "version_check"
377+version = "0.9.5"
378+source = "registry+https://github.com/rust-lang/crates.io-index"
379+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
380+
381+[[package]]
382+name = "wasm-bindgen"
383+version = "0.2.114"
384+source = "registry+https://github.com/rust-lang/crates.io-index"
385+checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e"
386+dependencies = [
387+ "cfg-if",
388+ "once_cell",
389+ "rustversion",
390+ "wasm-bindgen-macro",
391+ "wasm-bindgen-shared",
392+]
393+
394+[[package]]
395+name = "wasm-bindgen-macro"
396+version = "0.2.114"
397+source = "registry+https://github.com/rust-lang/crates.io-index"
398+checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6"
399+dependencies = [
400+ "quote",
401+ "wasm-bindgen-macro-support",
402+]
403+
404+[[package]]
405+name = "wasm-bindgen-macro-support"
406+version = "0.2.114"
407+source = "registry+https://github.com/rust-lang/crates.io-index"
408+checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3"
409+dependencies = [
410+ "bumpalo",
411+ "proc-macro2",
412+ "quote",
413+ "syn",
414+ "wasm-bindgen-shared",
415+]
416+
417+[[package]]
418+name = "wasm-bindgen-shared"
419+version = "0.2.114"
420+source = "registry+https://github.com/rust-lang/crates.io-index"
421+checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16"
422+dependencies = [
423+ "unicode-ident",
424+]
425+
426+[[package]]
427+name = "windows-link"
428+version = "0.2.1"
429+source = "registry+https://github.com/rust-lang/crates.io-index"
430+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
431+
432+[[package]]
433+name = "windows-sys"
434+version = "0.61.2"
435+source = "registry+https://github.com/rust-lang/crates.io-index"
436+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
437+dependencies = [
438+ "windows-link",
439+]
440+
441+[[package]]
442+name = "zmij"
443+version = "1.0.21"
444+source = "registry+https://github.com/rust-lang/crates.io-index"
445+checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
tools/openapi-extractor-rs/Cargo.toml+17−0
@@ -0,0 +1,17 @@
1+[package]
2+name = "openapi-extractor-rs"
3+version = "0.1.0"
4+edition = "2024"
5+
6+[lib]
7+crate-type = ["cdylib", "rlib"]
8+
9+[dependencies]
10+clap = { version = "4.5.40", features = ["derive"] }
11+serde = { version = "1.0.228", features = ["derive"] }
12+serde_json = "1.0.145"
13+serde_yaml = "0.9.34"
14+sha2 = "0.10.9"
15+
16+[target.'cfg(target_arch = "wasm32")'.dependencies]
17+wasm-bindgen = "0.2.114"
tools/openapi-extractor-rs/README.md+33−0
@@ -0,0 +1,33 @@
1+# openapi-extractor-rs
2+
3+Rust implementation of the OpenAPI extraction pipeline used by `packages/management-api/src/openapi-extraction.ts`.
4+
5+## Build
6+
7+```bash
8+cargo build --release
9+```
10+
11+## Build WASM Bindings For Runtime
12+
13+This is the production path used by `@executor-v2/management-api`.
14+
15+```bash
16+./build-wasm.sh
17+```
18+
19+The script compiles `wasm32-unknown-unknown` and writes bindings/artifacts to:
20+
21+- `packages/management-api/src/openapi-extractor-wasm/openapi_extractor.js`
22+- `packages/management-api/src/openapi-extractor-wasm/openapi_extractor_bg.wasm`
23+
24+## Run
25+
26+```bash
27+./target/release/openapi-extractor-rs \
28+ --source-name "Cloudflare API" \
29+ --input /tmp/cloudflare-openapi.yaml \
30+ --output /tmp/manifest-rs.json
31+```
32+
33+Use `--input -` to read the OpenAPI document from stdin. Use `--pretty` for formatted JSON output.
tools/openapi-extractor-rs/build-wasm.sh+20−0
@@ -0,0 +1,20 @@
1+#!/usr/bin/env bash
2+set -euo pipefail
3+
4+ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
5+CRATE_DIR="$ROOT_DIR/tools/openapi-extractor-rs"
6+OUT_DIR="$ROOT_DIR/packages/management-api/src/openapi-extractor-wasm"
7+
8+cd "$CRATE_DIR"
9+
10+rustup target add wasm32-unknown-unknown >/dev/null
11+
12+cargo build --release --target wasm32-unknown-unknown
13+
14+wasm-bindgen \
15+ --target web \
16+ --out-dir "$OUT_DIR" \
17+ --out-name openapi_extractor \
18+ "$CRATE_DIR/target/wasm32-unknown-unknown/release/openapi_extractor_rs.wasm"
19+
20+rm -rf "$CRATE_DIR/target"
tools/openapi-extractor-rs/src/lib.rs+690−0
@@ -0,0 +1,690 @@
1+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
2+
3+use serde::Serialize;
4+use serde_json::{Map, Value, json};
5+use sha2::{Digest, Sha256};
6+
7+#[cfg(target_arch = "wasm32")]
8+use wasm_bindgen::prelude::*;
9+
10+const OPEN_API_HTTP_METHODS: [&str; 8] = [
11+ "get",
12+ "put",
13+ "post",
14+ "delete",
15+ "patch",
16+ "head",
17+ "options",
18+ "trace",
19+];
20+
21+const OPEN_API_PARAMETER_LOCATIONS: [&str; 4] = ["path", "query", "header", "cookie"];
22+
23+#[derive(Serialize, Clone)]
24+struct OpenApiToolParameter {
25+ name: String,
26+ location: String,
27+ required: bool,
28+}
29+
30+#[derive(Serialize, Clone)]
31+#[serde(rename_all = "camelCase")]
32+struct OpenApiToolRequestBody {
33+ required: bool,
34+ content_types: Vec<String>,
35+}
36+
37+#[derive(Serialize, Clone)]
38+#[serde(rename_all = "camelCase")]
39+struct OpenApiInvocationPayload {
40+ method: String,
41+ path_template: String,
42+ parameters: Vec<OpenApiToolParameter>,
43+ request_body: Option<OpenApiToolRequestBody>,
44+}
45+
46+#[derive(Serialize, Clone)]
47+#[serde(rename_all = "camelCase")]
48+struct DiscoveryTypingPayload {
49+ #[serde(skip_serializing_if = "Option::is_none")]
50+ input_schema_json: Option<String>,
51+ #[serde(skip_serializing_if = "Option::is_none")]
52+ output_schema_json: Option<String>,
53+ #[serde(skip_serializing_if = "Option::is_none")]
54+ ref_hint_keys: Option<Vec<String>>,
55+}
56+
57+#[derive(Serialize, Clone)]
58+#[serde(rename_all = "camelCase")]
59+struct OpenApiExtractedTool {
60+ tool_id: String,
61+ name: String,
62+ description: Option<String>,
63+ method: String,
64+ path: String,
65+ invocation: OpenApiInvocationPayload,
66+ operation_hash: String,
67+ #[serde(skip_serializing_if = "Option::is_none")]
68+ typing: Option<DiscoveryTypingPayload>,
69+}
70+
71+#[derive(Serialize)]
72+#[serde(rename_all = "camelCase")]
73+struct OpenApiToolManifest {
74+ version: u8,
75+ source_hash: String,
76+ tools: Vec<OpenApiExtractedTool>,
77+ #[serde(skip_serializing_if = "Option::is_none")]
78+ ref_hint_table: Option<BTreeMap<String, String>>,
79+}
80+
81+pub fn extract_manifest_json(
82+ source_name: &str,
83+ openapi_document_text: &str,
84+ pretty: bool,
85+) -> Result<String, String> {
86+ let spec = parse_openapi_document(openapi_document_text)?;
87+ let manifest = extract_openapi_manifest(source_name, &spec)?;
88+
89+ if pretty {
90+ serde_json::to_string_pretty(&manifest).map_err(|error| format!("serialize manifest: {error}"))
91+ } else {
92+ serde_json::to_string(&manifest).map_err(|error| format!("serialize manifest: {error}"))
93+ }
94+}
95+
96+#[cfg(target_arch = "wasm32")]
97+#[wasm_bindgen]
98+pub fn extract_manifest_json_wasm(
99+ source_name: &str,
100+ openapi_document_text: &str,
101+) -> Result<String, JsValue> {
102+ extract_manifest_json(source_name, openapi_document_text, false)
103+ .map_err(|error| JsValue::from_str(&error))
104+}
105+
106+fn parse_openapi_document(input: &str) -> Result<Value, String> {
107+ let text = input.trim();
108+ if text.is_empty() {
109+ return Err("OpenAPI document is empty".to_string());
110+ }
111+
112+ match serde_json::from_str::<Value>(text) {
113+ Ok(value) => Ok(value),
114+ Err(_) => serde_yaml::from_str::<Value>(text)
115+ .map_err(|error| format!("Unable to parse OpenAPI document as JSON or YAML: {error}")),
116+ }
117+}
118+
119+fn as_object(value: &Value) -> Option<&Map<String, Value>> {
120+ value.as_object()
121+}
122+
123+fn as_trimmed_string(value: Option<&Value>) -> Option<String> {
124+ let candidate = value?.as_str()?.trim();
125+ if candidate.is_empty() {
126+ return None;
127+ }
128+
129+ Some(candidate.to_string())
130+}
131+
132+fn response_status_rank(status_code: &str) -> i32 {
133+ if status_code.len() == 3
134+ && status_code.starts_with('2')
135+ && status_code.chars().all(|character| character.is_ascii_digit())
136+ {
137+ return 0;
138+ }
139+
140+ if status_code == "default" {
141+ return 1;
142+ }
143+
144+ 2
145+}
146+
147+fn to_stable_value(value: &Value) -> Value {
148+ match value {
149+ Value::Array(items) => Value::Array(items.iter().map(to_stable_value).collect()),
150+ Value::Object(object) => {
151+ let mut keys: Vec<&String> = object.keys().collect();
152+ keys.sort();
153+
154+ let mut stable = Map::new();
155+ for key in keys {
156+ if let Some(item) = object.get(key) {
157+ stable.insert(key.clone(), to_stable_value(item));
158+ }
159+ }
160+
161+ Value::Object(stable)
162+ }
163+ Value::Number(number) => {
164+ if let Some(float_value) = number.as_f64()
165+ && float_value.is_finite()
166+ && float_value.fract() == 0.0
167+ {
168+ if float_value >= 0.0 && float_value <= u64::MAX as f64 {
169+ let unsigned = float_value as u64;
170+ if unsigned as f64 == float_value {
171+ return Value::Number(serde_json::Number::from(unsigned));
172+ }
173+ }
174+
175+ if float_value >= i64::MIN as f64 && float_value <= i64::MAX as f64 {
176+ let signed = float_value as i64;
177+ if signed as f64 == float_value {
178+ return Value::Number(serde_json::Number::from(signed));
179+ }
180+ }
181+ }
182+
183+ Value::Number(number.clone())
184+ }
185+ _ => value.clone(),
186+ }
187+}
188+
189+fn hash_unknown(value: &Value) -> Result<String, String> {
190+ let stable_json =
191+ serde_json::to_string(&to_stable_value(value)).map_err(|error| format!("hash json: {error}"))?;
192+
193+ let mut hasher = Sha256::new();
194+ hasher.update(stable_json.as_bytes());
195+ Ok(format!("{:x}", hasher.finalize()))
196+}
197+
198+fn encode_stable_json(value: &Value) -> Result<String, String> {
199+ serde_json::to_string(&to_stable_value(value)).map_err(|error| format!("encode stable json: {error}"))
200+}
201+
202+fn to_extracted_tool_parameter(value: &Value) -> Option<OpenApiToolParameter> {
203+ let record = as_object(value)?;
204+ let name = record.get("name")?.as_str()?.trim();
205+
206+ if name.is_empty() {
207+ return None;
208+ }
209+
210+ let location = record.get("in")?.as_str()?;
211+ if !OPEN_API_PARAMETER_LOCATIONS.contains(&location) {
212+ return None;
213+ }
214+
215+ let required = location == "path" || record.get("required").and_then(Value::as_bool) == Some(true);
216+
217+ Some(OpenApiToolParameter {
218+ name: name.to_string(),
219+ location: location.to_string(),
220+ required,
221+ })
222+}
223+
224+fn merge_parameters(path_item: &Map<String, Value>, operation: &Map<String, Value>) -> Vec<OpenApiToolParameter> {
225+ let mut by_key: HashMap<String, OpenApiToolParameter> = HashMap::new();
226+
227+ let mut add_parameters = |candidate: Option<&Value>| {
228+ let Some(items) = candidate.and_then(Value::as_array) else {
229+ return;
230+ };
231+
232+ for item in items {
233+ if let Some(parameter) = to_extracted_tool_parameter(item) {
234+ by_key.insert(
235+ format!("{}:{}", parameter.location, parameter.name),
236+ parameter,
237+ );
238+ }
239+ }
240+ };
241+
242+ add_parameters(path_item.get("parameters"));
243+ add_parameters(operation.get("parameters"));
244+
245+ let mut merged: Vec<OpenApiToolParameter> = by_key.into_values().collect();
246+ merged.sort_by(|left, right| {
247+ left.location
248+ .cmp(&right.location)
249+ .then_with(|| left.name.cmp(&right.name))
250+ });
251+ merged
252+}
253+
254+fn extract_request_body(operation: &Map<String, Value>) -> Option<OpenApiToolRequestBody> {
255+ let request_body = as_object(operation.get("requestBody")?)?;
256+ let content = request_body.get("content").and_then(as_object);
257+
258+ let mut content_types = content
259+ .map(|object| object.keys().cloned().collect::<Vec<String>>())
260+ .unwrap_or_default();
261+ content_types.sort();
262+
263+ Some(OpenApiToolRequestBody {
264+ required: request_body.get("required").and_then(Value::as_bool) == Some(true),
265+ content_types,
266+ })
267+}
268+
269+fn build_invocation_metadata(
270+ method: &str,
271+ path_value: &str,
272+ path_item: &Map<String, Value>,
273+ operation: &Map<String, Value>,
274+) -> OpenApiInvocationPayload {
275+ OpenApiInvocationPayload {
276+ method: method.to_string(),
277+ path_template: path_value.to_string(),
278+ parameters: merge_parameters(path_item, operation),
279+ request_body: extract_request_body(operation),
280+ }
281+}
282+
283+fn collect_ref_keys(value: &Value, refs: &mut BTreeSet<String>) {
284+ match value {
285+ Value::Array(items) => {
286+ for item in items {
287+ collect_ref_keys(item, refs);
288+ }
289+ }
290+ Value::Object(record) => {
291+ if let Some(reference) = record.get("$ref").and_then(Value::as_str)
292+ && reference.starts_with("#/")
293+ {
294+ refs.insert(reference.to_string());
295+ }
296+
297+ for nested in record.values() {
298+ collect_ref_keys(nested, refs);
299+ }
300+ }
301+ _ => {}
302+ }
303+}
304+
305+fn resolve_json_pointer<'a>(root: &'a Value, pointer: &str) -> Option<&'a Value> {
306+ if !pointer.starts_with("#/") {
307+ return None;
308+ }
309+
310+ let mut current = root;
311+ for part in pointer[2..].split('/') {
312+ let decoded = part.replace("~1", "/").replace("~0", "~");
313+ let next = current.as_object()?.get(&decoded)?;
314+ current = next;
315+ }
316+
317+ Some(current)
318+}
319+
320+fn pick_schema_from_content(content: Option<&Value>) -> Option<Value> {
321+ let content_record = as_object(content?)?;
322+
323+ let mut keys: Vec<String> = content_record.keys().cloned().collect();
324+ keys.sort();
325+
326+ let mut preferred = Vec::with_capacity(keys.len() + 1);
327+ preferred.push("application/json".to_string());
328+ preferred.extend(keys);
329+
330+ let mut seen = HashSet::new();
331+ for media_type in preferred {
332+ if !seen.insert(media_type.clone()) {
333+ continue;
334+ }
335+
336+ let Some(media_type_value) = content_record.get(&media_type).and_then(as_object) else {
337+ continue;
338+ };
339+
340+ if let Some(schema) = media_type_value.get("schema") {
341+ return Some(schema.clone());
342+ }
343+ }
344+
345+ None
346+}
347+
348+fn extract_request_body_schema(operation: &Map<String, Value>) -> Option<Value> {
349+ let request_body = as_object(operation.get("requestBody")?)?;
350+ pick_schema_from_content(request_body.get("content"))
351+}
352+
353+fn extract_response_schema(operation: &Map<String, Value>) -> Option<Value> {
354+ let responses = as_object(operation.get("responses")?)?;
355+ let mut response_codes: Vec<String> = responses.keys().cloned().collect();
356+ response_codes.sort_by(|left, right| {
357+ response_status_rank(left)
358+ .cmp(&response_status_rank(right))
359+ .then_with(|| left.cmp(right))
360+ });
361+
362+ for response_code in response_codes {
363+ let Some(response) = responses.get(&response_code).and_then(as_object) else {
364+ continue;
365+ };
366+
367+ let schema = pick_schema_from_content(response.get("content"));
368+ if schema.is_some() {
369+ return schema;
370+ }
371+ }
372+
373+ None
374+}
375+
376+fn collect_parameter_schema_by_key(
377+ path_item: &Map<String, Value>,
378+ operation: &Map<String, Value>,
379+) -> HashMap<String, Value> {
380+ let mut schemas = HashMap::new();
381+
382+ let mut add_parameters = |candidate: Option<&Value>| {
383+ let Some(items) = candidate.and_then(Value::as_array) else {
384+ return;
385+ };
386+
387+ for item in items {
388+ let Some(item_record) = as_object(item) else {
389+ continue;
390+ };
391+
392+ let Some(parameter) = to_extracted_tool_parameter(item) else {
393+ continue;
394+ };
395+
396+ let Some(schema) = item_record.get("schema") else {
397+ continue;
398+ };
399+
400+ schemas.insert(
401+ format!("{}:{}", parameter.location, parameter.name),
402+ schema.clone(),
403+ );
404+ }
405+ };
406+
407+ add_parameters(path_item.get("parameters"));
408+ add_parameters(operation.get("parameters"));
409+
410+ schemas
411+}
412+
413+fn build_input_schema(
414+ path_item: &Map<String, Value>,
415+ operation: &Map<String, Value>,
416+ invocation: &OpenApiInvocationPayload,
417+) -> Option<Value> {
418+ let parameter_schema_by_key = collect_parameter_schema_by_key(path_item, operation);
419+
420+ let mut properties = Map::new();
421+ let mut required = BTreeSet::new();
422+
423+ for parameter in &invocation.parameters {
424+ let key = format!("{}:{}", parameter.location, parameter.name);
425+ let schema = parameter_schema_by_key
426+ .get(&key)
427+ .cloned()
428+ .unwrap_or_else(|| json!({ "type": "string" }));
429+
430+ properties.insert(parameter.name.clone(), schema);
431+
432+ if parameter.required {
433+ required.insert(parameter.name.clone());
434+ }
435+ }
436+
437+ let request_body_schema = extract_request_body_schema(operation);
438+ if let Some(schema) = request_body_schema {
439+ properties.insert("body".to_string(), schema);
440+
441+ if invocation
442+ .request_body
443+ .as_ref()
444+ .map(|request_body| request_body.required)
445+ == Some(true)
446+ {
447+ required.insert("body".to_string());
448+ }
449+ }
450+
451+ if properties.is_empty() {
452+ return None;
453+ }
454+
455+ let required_list: Vec<String> = required.into_iter().collect();
456+
457+ Some(json!({
458+ "type": "object",
459+ "properties": properties,
460+ "required": required_list,
461+ "additionalProperties": false,
462+ }))
463+}
464+
465+fn build_tool_typing(
466+ path_item: &Map<String, Value>,
467+ operation: &Map<String, Value>,
468+ invocation: &OpenApiInvocationPayload,
469+) -> Result<Option<DiscoveryTypingPayload>, String> {
470+ let input_schema = build_input_schema(path_item, operation, invocation);
471+ let output_schema = extract_response_schema(operation);
472+
473+ if input_schema.is_none() && output_schema.is_none() {
474+ return Ok(None);
475+ }
476+
477+ let mut refs = BTreeSet::new();
478+ if let Some(schema) = &input_schema {
479+ collect_ref_keys(schema, &mut refs);
480+ }
481+
482+ if let Some(schema) = &output_schema {
483+ collect_ref_keys(schema, &mut refs);
484+ }
485+
486+ let ref_hint_keys = if refs.is_empty() {
487+ None
488+ } else {
489+ Some(refs.into_iter().collect())
490+ };
491+
492+ Ok(Some(DiscoveryTypingPayload {
493+ input_schema_json: input_schema
494+ .as_ref()
495+ .map(encode_stable_json)
496+ .transpose()?,
497+ output_schema_json: output_schema
498+ .as_ref()
499+ .map(encode_stable_json)
500+ .transpose()?,
501+ ref_hint_keys,
502+ }))
503+}
504+
505+fn build_ref_hint_table(open_api_spec: &Value, initial_ref_keys: &[String]) -> Result<BTreeMap<String, String>, String> {
506+ let mut queue = VecDeque::new();
507+ let mut queued = HashSet::new();
508+
509+ for ref_key in initial_ref_keys {
510+ if queued.insert(ref_key.clone()) {
511+ queue.push_back(ref_key.clone());
512+ }
513+ }
514+
515+ let mut seen = HashSet::new();
516+ let mut table = BTreeMap::new();
517+
518+ while let Some(ref_key) = queue.pop_front() {
519+ if !seen.insert(ref_key.clone()) {
520+ continue;
521+ }
522+
523+ let Some(resolved) = resolve_json_pointer(open_api_spec, &ref_key) else {
524+ continue;
525+ };
526+
527+ table.insert(ref_key.clone(), encode_stable_json(resolved)?);
528+
529+ let mut nested = BTreeSet::new();
530+ collect_ref_keys(resolved, &mut nested);
531+ for nested_ref in nested {
532+ if !seen.contains(&nested_ref) {
533+ queue.push_back(nested_ref);
534+ }
535+ }
536+ }
537+
538+ Ok(table)
539+}
540+
541+fn normalize_path_for_tool_id(path_value: &str) -> String {
542+ let trimmed = path_value.trim();
543+ let no_leading = trimmed.strip_prefix('/').unwrap_or(trimmed);
544+
545+ let mut normalized = String::new();
546+ let mut last_was_underscore = false;
547+
548+ for character in no_leading.chars() {
549+ if character.is_ascii_alphanumeric() {
550+ normalized.push(character);
551+ last_was_underscore = false;
552+ continue;
553+ }
554+
555+ if !last_was_underscore {
556+ normalized.push('_');
557+ last_was_underscore = true;
558+ }
559+ }
560+
561+ let without_edges = normalized.trim_matches('_').to_lowercase();
562+ if without_edges.is_empty() {
563+ return "root".to_string();
564+ }
565+
566+ without_edges
567+}
568+
569+fn build_tool_id(method: &str, path_value: &str, operation: &Map<String, Value>) -> String {
570+ if let Some(operation_id) = as_trimmed_string(operation.get("operationId")) {
571+ return operation_id;
572+ }
573+
574+ format!("{}_{}", method, normalize_path_for_tool_id(path_value))
575+}
576+
577+fn build_tool_name(method: &str, path_value: &str, operation: &Map<String, Value>) -> String {
578+ if let Some(summary) = as_trimmed_string(operation.get("summary")) {
579+ return summary;
580+ }
581+
582+ if let Some(operation_id) = as_trimmed_string(operation.get("operationId")) {
583+ return operation_id;
584+ }
585+
586+ format!("{} {}", method.to_uppercase(), path_value)
587+}
588+
589+fn build_tool_description(operation: &Map<String, Value>) -> Option<String> {
590+ as_trimmed_string(operation.get("description")).or_else(|| as_trimmed_string(operation.get("summary")))
591+}
592+
593+fn ensure_unique_tool_ids(source_name: &str, tools: &[OpenApiExtractedTool]) -> Result<(), String> {
594+ let mut seen = HashSet::new();
595+
596+ for tool in tools {
597+ if !seen.insert(tool.tool_id.clone()) {
598+ return Err(format!(
599+ "OpenAPI extraction failed for '{source_name}': duplicate toolId {} ({})",
600+ tool.tool_id,
601+ tool.path
602+ ));
603+ }
604+ }
605+
606+ Ok(())
607+}
608+
609+fn extract_openapi_manifest(source_name: &str, openapi_spec: &Value) -> Result<OpenApiToolManifest, String> {
610+ let spec_record = as_object(openapi_spec)
611+ .ok_or_else(|| format!("OpenAPI extraction failed for '{source_name}': spec must be object"))?;
612+
613+ let Some(paths) = spec_record.get("paths").and_then(as_object) else {
614+ return Ok(OpenApiToolManifest {
615+ version: 1,
616+ source_hash: hash_unknown(openapi_spec)?,
617+ tools: Vec::new(),
618+ ref_hint_table: None,
619+ });
620+ };
621+
622+ let mut path_keys: Vec<String> = paths.keys().cloned().collect();
623+ path_keys.sort();
624+
625+ let mut tools = Vec::new();
626+
627+ for path_value in path_keys {
628+ let Some(path_item) = paths.get(&path_value).and_then(as_object) else {
629+ continue;
630+ };
631+
632+ for method in OPEN_API_HTTP_METHODS {
633+ let Some(operation) = path_item.get(method).and_then(as_object) else {
634+ continue;
635+ };
636+
637+ let invocation = build_invocation_metadata(method, &path_value, path_item, operation);
638+ let typing = build_tool_typing(path_item, operation, &invocation)?;
639+
640+ let operation_hash_input = json!({
641+ "method": method,
642+ "path": path_value,
643+ "operation": operation,
644+ "invocation": invocation,
645+ });
646+
647+ tools.push(OpenApiExtractedTool {
648+ tool_id: build_tool_id(method, &path_value, operation),
649+ name: build_tool_name(method, &path_value, operation),
650+ description: build_tool_description(operation),
651+ method: method.to_string(),
652+ path: path_value.clone(),
653+ invocation,
654+ operation_hash: hash_unknown(&operation_hash_input)?,
655+ typing,
656+ });
657+ }
658+ }
659+
660+ tools.sort_by(|left, right| {
661+ left.tool_id
662+ .to_lowercase()
663+ .cmp(&right.tool_id.to_lowercase())
664+ .then_with(|| left.tool_id.cmp(&right.tool_id))
665+ });
666+ ensure_unique_tool_ids(source_name, &tools)?;
667+
668+ let direct_ref_keys: Vec<String> = tools
669+ .iter()
670+ .flat_map(|tool| {
671+ tool.typing
672+ .as_ref()
673+ .and_then(|typing| typing.ref_hint_keys.clone())
674+ .unwrap_or_default()
675+ })
676+ .collect();
677+
678+ let ref_hint_table = build_ref_hint_table(openapi_spec, &direct_ref_keys)?;
679+
680+ Ok(OpenApiToolManifest {
681+ version: 1,
682+ source_hash: hash_unknown(openapi_spec)?,
683+ tools,
684+ ref_hint_table: if ref_hint_table.is_empty() {
685+ None
686+ } else {
687+ Some(ref_hint_table)
688+ },
689+ })
690+}
tools/openapi-extractor-rs/src/main.rs+58−0
@@ -0,0 +1,58 @@
1+use std::fs;
2+use std::io::{self, Read, Write};
3+
4+use clap::Parser;
5+
6+#[derive(Parser, Debug)]
7+#[command(name = "openapi-extractor-rs")]
8+struct Cli {
9+ #[arg(long)]
10+ source_name: String,
11+ #[arg(long)]
12+ input: String,
13+ #[arg(long)]
14+ output: Option<String>,
15+ #[arg(long, default_value_t = false)]
16+ pretty: bool,
17+}
18+
19+fn main() {
20+ if let Err(error) = run() {
21+ eprintln!("{error}");
22+ std::process::exit(1);
23+ }
24+}
25+
26+fn run() -> Result<(), String> {
27+ let args = Cli::parse();
28+ let source_text = read_input_text(&args.input)?;
29+
30+ let output_json =
31+ openapi_extractor_rs::extract_manifest_json(&args.source_name, &source_text, args.pretty)?;
32+
33+ if let Some(path) = args.output {
34+ fs::write(path, output_json).map_err(|error| format!("write output file: {error}"))?;
35+ } else {
36+ let mut stdout = io::stdout().lock();
37+ stdout
38+ .write_all(output_json.as_bytes())
39+ .map_err(|error| format!("write stdout: {error}"))?;
40+ stdout
41+ .write_all(b"\n")
42+ .map_err(|error| format!("write stdout newline: {error}"))?;
43+ }
44+
45+ Ok(())
46+}
47+
48+fn read_input_text(input: &str) -> Result<String, String> {
49+ if input == "-" {
50+ let mut buffer = String::new();
51+ io::stdin()
52+ .read_to_string(&mut buffer)
53+ .map_err(|error| format!("read stdin: {error}"))?;
54+ return Ok(buffer);
55+ }
56+
57+ fs::read_to_string(input).map_err(|error| format!("read input file '{input}': {error}"))
58+}