History

feat(v2): replace OpenAPI extraction with Rust WASM

Move OpenAPI extraction to a single Rust-backed WebAssembly path so large specs process much faster while keeping one maintainable runtime flow across environments.

Rhys Sullivan committed 8 months ago

Commit 7213d701 parent 322f1b916 files · +1644 −628Browse files

Showing the changes to tools (7 of 16 files). Show all

Unified · Split

7 files
  1. tools/openapi-extractor-rs/.gitignore +1 −0
  2. tools/openapi-extractor-rs/Cargo.lock +445 −0
  3. tools/openapi-extractor-rs/Cargo.toml +17 −0
  4. tools/openapi-extractor-rs/README.md +33 −0
  5. tools/openapi-extractor-rs/build-wasm.sh +20 −0
  6. tools/openapi-extractor-rs/src/lib.rs +690 −0
  7. tools/openapi-extractor-rs/src/main.rs +58 −0
tools/openapi-extractor-rs/.gitignore+1−0
@@ -0,0 +1 @@
1target/
tools/openapi-extractor-rs/Cargo.lock+445−0
@@ -0,0 +1,445 @@
1# This file is automatically @generated by Cargo.
2# It is not intended for manual editing.
3version = 4
4
5[[package]]
6name = "anstream"
7version = "0.6.21"
8source = "registry+https://github.com/rust-lang/crates.io-index"
9checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
10dependencies = [
11 "anstyle",
12 "anstyle-parse",
13 "anstyle-query",
14 "anstyle-wincon",
15 "colorchoice",
16 "is_terminal_polyfill",
17 "utf8parse",
18]
19
20[[package]]
21name = "anstyle"
22version = "1.0.13"
23source = "registry+https://github.com/rust-lang/crates.io-index"
24checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
25
26[[package]]
27name = "anstyle-parse"
28version = "0.2.7"
29source = "registry+https://github.com/rust-lang/crates.io-index"
30checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
31dependencies = [
32 "utf8parse",
33]
34
35[[package]]
36name = "anstyle-query"
37version = "1.1.5"
38source = "registry+https://github.com/rust-lang/crates.io-index"
39checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
40dependencies = [
41 "windows-sys",
42]
43
44[[package]]
45name = "anstyle-wincon"
46version = "3.0.11"
47source = "registry+https://github.com/rust-lang/crates.io-index"
48checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
49dependencies = [
50 "anstyle",
51 "once_cell_polyfill",
52 "windows-sys",
53]
54
55[[package]]
56name = "block-buffer"
57version = "0.10.4"
58source = "registry+https://github.com/rust-lang/crates.io-index"
59checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
60dependencies = [
61 "generic-array",
62]
63
64[[package]]
65name = "bumpalo"
66version = "3.20.2"
67source = "registry+https://github.com/rust-lang/crates.io-index"
68checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb"
69
70[[package]]
71name = "cfg-if"
72version = "1.0.4"
73source = "registry+https://github.com/rust-lang/crates.io-index"
74checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
75
76[[package]]
77name = "clap"
78version = "4.5.60"
79source = "registry+https://github.com/rust-lang/crates.io-index"
80checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a"
81dependencies = [
82 "clap_builder",
83 "clap_derive",
84]
85
86[[package]]
87name = "clap_builder"
88version = "4.5.60"
89source = "registry+https://github.com/rust-lang/crates.io-index"
90checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876"
91dependencies = [
92 "anstream",
93 "anstyle",
94 "clap_lex",
95 "strsim",
96]
97
98[[package]]
99name = "clap_derive"
100version = "4.5.55"
101source = "registry+https://github.com/rust-lang/crates.io-index"
102checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5"
103dependencies = [
104 "heck",
105 "proc-macro2",
106 "quote",
107 "syn",
108]
109
110[[package]]
111name = "clap_lex"
112version = "1.0.0"
113source = "registry+https://github.com/rust-lang/crates.io-index"
114checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831"
115
116[[package]]
117name = "colorchoice"
118version = "1.0.4"
119source = "registry+https://github.com/rust-lang/crates.io-index"
120checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
121
122[[package]]
123name = "cpufeatures"
124version = "0.2.17"
125source = "registry+https://github.com/rust-lang/crates.io-index"
126checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
127dependencies = [
128 "libc",
129]
130
131[[package]]
132name = "crypto-common"
133version = "0.1.7"
134source = "registry+https://github.com/rust-lang/crates.io-index"
135checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
136dependencies = [
137 "generic-array",
138 "typenum",
139]
140
141[[package]]
142name = "digest"
143version = "0.10.7"
144source = "registry+https://github.com/rust-lang/crates.io-index"
145checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
146dependencies = [
147 "block-buffer",
148 "crypto-common",
149]
150
151[[package]]
152name = "equivalent"
153version = "1.0.2"
154source = "registry+https://github.com/rust-lang/crates.io-index"
155checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
156
157[[package]]
158name = "generic-array"
159version = "0.14.7"
160source = "registry+https://github.com/rust-lang/crates.io-index"
161checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
162dependencies = [
163 "typenum",
164 "version_check",
165]
166
167[[package]]
168name = "hashbrown"
169version = "0.16.1"
170source = "registry+https://github.com/rust-lang/crates.io-index"
171checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
172
173[[package]]
174name = "heck"
175version = "0.5.0"
176source = "registry+https://github.com/rust-lang/crates.io-index"
177checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
178
179[[package]]
180name = "indexmap"
181version = "2.13.0"
182source = "registry+https://github.com/rust-lang/crates.io-index"
183checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017"
184dependencies = [
185 "equivalent",
186 "hashbrown",
187]
188
189[[package]]
190name = "is_terminal_polyfill"
191version = "1.70.2"
192source = "registry+https://github.com/rust-lang/crates.io-index"
193checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
194
195[[package]]
196name = "itoa"
197version = "1.0.17"
198source = "registry+https://github.com/rust-lang/crates.io-index"
199checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2"
200
201[[package]]
202name = "libc"
203version = "0.2.182"
204source = "registry+https://github.com/rust-lang/crates.io-index"
205checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112"
206
207[[package]]
208name = "memchr"
209version = "2.8.0"
210source = "registry+https://github.com/rust-lang/crates.io-index"
211checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79"
212
213[[package]]
214name = "once_cell"
215version = "1.21.3"
216source = "registry+https://github.com/rust-lang/crates.io-index"
217checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
218
219[[package]]
220name = "once_cell_polyfill"
221version = "1.70.2"
222source = "registry+https://github.com/rust-lang/crates.io-index"
223checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
224
225[[package]]
226name = "openapi-extractor-rs"
227version = "0.1.0"
228dependencies = [
229 "clap",
230 "serde",
231 "serde_json",
232 "serde_yaml",
233 "sha2",
234 "wasm-bindgen",
235]
236
237[[package]]
238name = "proc-macro2"
239version = "1.0.106"
240source = "registry+https://github.com/rust-lang/crates.io-index"
241checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
242dependencies = [
243 "unicode-ident",
244]
245
246[[package]]
247name = "quote"
248version = "1.0.44"
249source = "registry+https://github.com/rust-lang/crates.io-index"
250checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
251dependencies = [
252 "proc-macro2",
253]
254
255[[package]]
256name = "rustversion"
257version = "1.0.22"
258source = "registry+https://github.com/rust-lang/crates.io-index"
259checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
260
261[[package]]
262name = "ryu"
263version = "1.0.23"
264source = "registry+https://github.com/rust-lang/crates.io-index"
265checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
266
267[[package]]
268name = "serde"
269version = "1.0.228"
270source = "registry+https://github.com/rust-lang/crates.io-index"
271checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
272dependencies = [
273 "serde_core",
274 "serde_derive",
275]
276
277[[package]]
278name = "serde_core"
279version = "1.0.228"
280source = "registry+https://github.com/rust-lang/crates.io-index"
281checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
282dependencies = [
283 "serde_derive",
284]
285
286[[package]]
287name = "serde_derive"
288version = "1.0.228"
289source = "registry+https://github.com/rust-lang/crates.io-index"
290checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
291dependencies = [
292 "proc-macro2",
293 "quote",
294 "syn",
295]
296
297[[package]]
298name = "serde_json"
299version = "1.0.149"
300source = "registry+https://github.com/rust-lang/crates.io-index"
301checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86"
302dependencies = [
303 "itoa",
304 "memchr",
305 "serde",
306 "serde_core",
307 "zmij",
308]
309
310[[package]]
311name = "serde_yaml"
312version = "0.9.34+deprecated"
313source = "registry+https://github.com/rust-lang/crates.io-index"
314checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
315dependencies = [
316 "indexmap",
317 "itoa",
318 "ryu",
319 "serde",
320 "unsafe-libyaml",
321]
322
323[[package]]
324name = "sha2"
325version = "0.10.9"
326source = "registry+https://github.com/rust-lang/crates.io-index"
327checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
328dependencies = [
329 "cfg-if",
330 "cpufeatures",
331 "digest",
332]
333
334[[package]]
335name = "strsim"
336version = "0.11.1"
337source = "registry+https://github.com/rust-lang/crates.io-index"
338checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
339
340[[package]]
341name = "syn"
342version = "2.0.117"
343source = "registry+https://github.com/rust-lang/crates.io-index"
344checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99"
345dependencies = [
346 "proc-macro2",
347 "quote",
348 "unicode-ident",
349]
350
351[[package]]
352name = "typenum"
353version = "1.19.0"
354source = "registry+https://github.com/rust-lang/crates.io-index"
355checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
356
357[[package]]
358name = "unicode-ident"
359version = "1.0.24"
360source = "registry+https://github.com/rust-lang/crates.io-index"
361checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
362
363[[package]]
364name = "unsafe-libyaml"
365version = "0.2.11"
366source = "registry+https://github.com/rust-lang/crates.io-index"
367checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
368
369[[package]]
370name = "utf8parse"
371version = "0.2.2"
372source = "registry+https://github.com/rust-lang/crates.io-index"
373checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
374
375[[package]]
376name = "version_check"
377version = "0.9.5"
378source = "registry+https://github.com/rust-lang/crates.io-index"
379checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
380
381[[package]]
382name = "wasm-bindgen"
383version = "0.2.114"
384source = "registry+https://github.com/rust-lang/crates.io-index"
385checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e"
386dependencies = [
387 "cfg-if",
388 "once_cell",
389 "rustversion",
390 "wasm-bindgen-macro",
391 "wasm-bindgen-shared",
392]
393
394[[package]]
395name = "wasm-bindgen-macro"
396version = "0.2.114"
397source = "registry+https://github.com/rust-lang/crates.io-index"
398checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6"
399dependencies = [
400 "quote",
401 "wasm-bindgen-macro-support",
402]
403
404[[package]]
405name = "wasm-bindgen-macro-support"
406version = "0.2.114"
407source = "registry+https://github.com/rust-lang/crates.io-index"
408checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3"
409dependencies = [
410 "bumpalo",
411 "proc-macro2",
412 "quote",
413 "syn",
414 "wasm-bindgen-shared",
415]
416
417[[package]]
418name = "wasm-bindgen-shared"
419version = "0.2.114"
420source = "registry+https://github.com/rust-lang/crates.io-index"
421checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16"
422dependencies = [
423 "unicode-ident",
424]
425
426[[package]]
427name = "windows-link"
428version = "0.2.1"
429source = "registry+https://github.com/rust-lang/crates.io-index"
430checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
431
432[[package]]
433name = "windows-sys"
434version = "0.61.2"
435source = "registry+https://github.com/rust-lang/crates.io-index"
436checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
437dependencies = [
438 "windows-link",
439]
440
441[[package]]
442name = "zmij"
443version = "1.0.21"
444source = "registry+https://github.com/rust-lang/crates.io-index"
445checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa"
tools/openapi-extractor-rs/Cargo.toml+17−0
@@ -0,0 +1,17 @@
1[package]
2name = "openapi-extractor-rs"
3version = "0.1.0"
4edition = "2024"
5
6[lib]
7crate-type = ["cdylib", "rlib"]
8
9[dependencies]
10clap = { version = "4.5.40", features = ["derive"] }
11serde = { version = "1.0.228", features = ["derive"] }
12serde_json = "1.0.145"
13serde_yaml = "0.9.34"
14sha2 = "0.10.9"
15
16[target.'cfg(target_arch = "wasm32")'.dependencies]
17wasm-bindgen = "0.2.114"
tools/openapi-extractor-rs/README.md+33−0
@@ -0,0 +1,33 @@
1# openapi-extractor-rs
2
3Rust implementation of the OpenAPI extraction pipeline used by `packages/management-api/src/openapi-extraction.ts`.
4
5## Build
6
7```bash
8cargo build --release
9```
10
11## Build WASM Bindings For Runtime
12
13This is the production path used by `@executor-v2/management-api`.
14
15```bash
16./build-wasm.sh
17```
18
19The script compiles `wasm32-unknown-unknown` and writes bindings/artifacts to:
20
21- `packages/management-api/src/openapi-extractor-wasm/openapi_extractor.js`
22- `packages/management-api/src/openapi-extractor-wasm/openapi_extractor_bg.wasm`
23
24## Run
25
26```bash
27./target/release/openapi-extractor-rs \
28 --source-name "Cloudflare API" \
29 --input /tmp/cloudflare-openapi.yaml \
30 --output /tmp/manifest-rs.json
31```
32
33Use `--input -` to read the OpenAPI document from stdin. Use `--pretty` for formatted JSON output.
tools/openapi-extractor-rs/build-wasm.sh+20−0
@@ -0,0 +1,20 @@
1#!/usr/bin/env bash
2set -euo pipefail
3
4ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
5CRATE_DIR="$ROOT_DIR/tools/openapi-extractor-rs"
6OUT_DIR="$ROOT_DIR/packages/management-api/src/openapi-extractor-wasm"
7
8cd "$CRATE_DIR"
9
10rustup target add wasm32-unknown-unknown >/dev/null
11
12cargo build --release --target wasm32-unknown-unknown
13
14wasm-bindgen \
15 --target web \
16 --out-dir "$OUT_DIR" \
17 --out-name openapi_extractor \
18 "$CRATE_DIR/target/wasm32-unknown-unknown/release/openapi_extractor_rs.wasm"
19
20rm -rf "$CRATE_DIR/target"
tools/openapi-extractor-rs/src/lib.rs+690−0
@@ -0,0 +1,690 @@
1use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet, VecDeque};
2
3use serde::Serialize;
4use serde_json::{Map, Value, json};
5use sha2::{Digest, Sha256};
6
7#[cfg(target_arch = "wasm32")]
8use wasm_bindgen::prelude::*;
9
10const OPEN_API_HTTP_METHODS: [&str; 8] = [
11 "get",
12 "put",
13 "post",
14 "delete",
15 "patch",
16 "head",
17 "options",
18 "trace",
19];
20
21const OPEN_API_PARAMETER_LOCATIONS: [&str; 4] = ["path", "query", "header", "cookie"];
22
23#[derive(Serialize, Clone)]
24struct OpenApiToolParameter {
25 name: String,
26 location: String,
27 required: bool,
28}
29
30#[derive(Serialize, Clone)]
31#[serde(rename_all = "camelCase")]
32struct OpenApiToolRequestBody {
33 required: bool,
34 content_types: Vec<String>,
35}
36
37#[derive(Serialize, Clone)]
38#[serde(rename_all = "camelCase")]
39struct OpenApiInvocationPayload {
40 method: String,
41 path_template: String,
42 parameters: Vec<OpenApiToolParameter>,
43 request_body: Option<OpenApiToolRequestBody>,
44}
45
46#[derive(Serialize, Clone)]
47#[serde(rename_all = "camelCase")]
48struct DiscoveryTypingPayload {
49 #[serde(skip_serializing_if = "Option::is_none")]
50 input_schema_json: Option<String>,
51 #[serde(skip_serializing_if = "Option::is_none")]
52 output_schema_json: Option<String>,
53 #[serde(skip_serializing_if = "Option::is_none")]
54 ref_hint_keys: Option<Vec<String>>,
55}
56
57#[derive(Serialize, Clone)]
58#[serde(rename_all = "camelCase")]
59struct OpenApiExtractedTool {
60 tool_id: String,
61 name: String,
62 description: Option<String>,
63 method: String,
64 path: String,
65 invocation: OpenApiInvocationPayload,
66 operation_hash: String,
67 #[serde(skip_serializing_if = "Option::is_none")]
68 typing: Option<DiscoveryTypingPayload>,
69}
70
71#[derive(Serialize)]
72#[serde(rename_all = "camelCase")]
73struct OpenApiToolManifest {
74 version: u8,
75 source_hash: String,
76 tools: Vec<OpenApiExtractedTool>,
77 #[serde(skip_serializing_if = "Option::is_none")]
78 ref_hint_table: Option<BTreeMap<String, String>>,
79}
80
81pub fn extract_manifest_json(
82 source_name: &str,
83 openapi_document_text: &str,
84 pretty: bool,
85) -> Result<String, String> {
86 let spec = parse_openapi_document(openapi_document_text)?;
87 let manifest = extract_openapi_manifest(source_name, &spec)?;
88
89 if pretty {
90 serde_json::to_string_pretty(&manifest).map_err(|error| format!("serialize manifest: {error}"))
91 } else {
92 serde_json::to_string(&manifest).map_err(|error| format!("serialize manifest: {error}"))
93 }
94}
95
96#[cfg(target_arch = "wasm32")]
97#[wasm_bindgen]
98pub fn extract_manifest_json_wasm(
99 source_name: &str,
100 openapi_document_text: &str,
101) -> Result<String, JsValue> {
102 extract_manifest_json(source_name, openapi_document_text, false)
103 .map_err(|error| JsValue::from_str(&error))
104}
105
106fn parse_openapi_document(input: &str) -> Result<Value, String> {
107 let text = input.trim();
108 if text.is_empty() {
109 return Err("OpenAPI document is empty".to_string());
110 }
111
112 match serde_json::from_str::<Value>(text) {
113 Ok(value) => Ok(value),
114 Err(_) => serde_yaml::from_str::<Value>(text)
115 .map_err(|error| format!("Unable to parse OpenAPI document as JSON or YAML: {error}")),
116 }
117}
118
119fn as_object(value: &Value) -> Option<&Map<String, Value>> {
120 value.as_object()
121}
122
123fn as_trimmed_string(value: Option<&Value>) -> Option<String> {
124 let candidate = value?.as_str()?.trim();
125 if candidate.is_empty() {
126 return None;
127 }
128
129 Some(candidate.to_string())
130}
131
132fn response_status_rank(status_code: &str) -> i32 {
133 if status_code.len() == 3
134 && status_code.starts_with('2')
135 && status_code.chars().all(|character| character.is_ascii_digit())
136 {
137 return 0;
138 }
139
140 if status_code == "default" {
141 return 1;
142 }
143
144 2
145}
146
147fn to_stable_value(value: &Value) -> Value {
148 match value {
149 Value::Array(items) => Value::Array(items.iter().map(to_stable_value).collect()),
150 Value::Object(object) => {
151 let mut keys: Vec<&String> = object.keys().collect();
152 keys.sort();
153
154 let mut stable = Map::new();
155 for key in keys {
156 if let Some(item) = object.get(key) {
157 stable.insert(key.clone(), to_stable_value(item));
158 }
159 }
160
161 Value::Object(stable)
162 }
163 Value::Number(number) => {
164 if let Some(float_value) = number.as_f64()
165 && float_value.is_finite()
166 && float_value.fract() == 0.0
167 {
168 if float_value >= 0.0 && float_value <= u64::MAX as f64 {
169 let unsigned = float_value as u64;
170 if unsigned as f64 == float_value {
171 return Value::Number(serde_json::Number::from(unsigned));
172 }
173 }
174
175 if float_value >= i64::MIN as f64 && float_value <= i64::MAX as f64 {
176 let signed = float_value as i64;
177 if signed as f64 == float_value {
178 return Value::Number(serde_json::Number::from(signed));
179 }
180 }
181 }
182
183 Value::Number(number.clone())
184 }
185 _ => value.clone(),
186 }
187}
188
189fn hash_unknown(value: &Value) -> Result<String, String> {
190 let stable_json =
191 serde_json::to_string(&to_stable_value(value)).map_err(|error| format!("hash json: {error}"))?;
192
193 let mut hasher = Sha256::new();
194 hasher.update(stable_json.as_bytes());
195 Ok(format!("{:x}", hasher.finalize()))
196}
197
198fn encode_stable_json(value: &Value) -> Result<String, String> {
199 serde_json::to_string(&to_stable_value(value)).map_err(|error| format!("encode stable json: {error}"))
200}
201
202fn to_extracted_tool_parameter(value: &Value) -> Option<OpenApiToolParameter> {
203 let record = as_object(value)?;
204 let name = record.get("name")?.as_str()?.trim();
205
206 if name.is_empty() {
207 return None;
208 }
209
210 let location = record.get("in")?.as_str()?;
211 if !OPEN_API_PARAMETER_LOCATIONS.contains(&location) {
212 return None;
213 }
214
215 let required = location == "path" || record.get("required").and_then(Value::as_bool) == Some(true);
216
217 Some(OpenApiToolParameter {
218 name: name.to_string(),
219 location: location.to_string(),
220 required,
221 })
222}
223
224fn merge_parameters(path_item: &Map<String, Value>, operation: &Map<String, Value>) -> Vec<OpenApiToolParameter> {
225 let mut by_key: HashMap<String, OpenApiToolParameter> = HashMap::new();
226
227 let mut add_parameters = |candidate: Option<&Value>| {
228 let Some(items) = candidate.and_then(Value::as_array) else {
229 return;
230 };
231
232 for item in items {
233 if let Some(parameter) = to_extracted_tool_parameter(item) {
234 by_key.insert(
235 format!("{}:{}", parameter.location, parameter.name),
236 parameter,
237 );
238 }
239 }
240 };
241
242 add_parameters(path_item.get("parameters"));
243 add_parameters(operation.get("parameters"));
244
245 let mut merged: Vec<OpenApiToolParameter> = by_key.into_values().collect();
246 merged.sort_by(|left, right| {
247 left.location
248 .cmp(&right.location)
249 .then_with(|| left.name.cmp(&right.name))
250 });
251 merged
252}
253
254fn extract_request_body(operation: &Map<String, Value>) -> Option<OpenApiToolRequestBody> {
255 let request_body = as_object(operation.get("requestBody")?)?;
256 let content = request_body.get("content").and_then(as_object);
257
258 let mut content_types = content
259 .map(|object| object.keys().cloned().collect::<Vec<String>>())
260 .unwrap_or_default();
261 content_types.sort();
262
263 Some(OpenApiToolRequestBody {
264 required: request_body.get("required").and_then(Value::as_bool) == Some(true),
265 content_types,
266 })
267}
268
269fn build_invocation_metadata(
270 method: &str,
271 path_value: &str,
272 path_item: &Map<String, Value>,
273 operation: &Map<String, Value>,
274) -> OpenApiInvocationPayload {
275 OpenApiInvocationPayload {
276 method: method.to_string(),
277 path_template: path_value.to_string(),
278 parameters: merge_parameters(path_item, operation),
279 request_body: extract_request_body(operation),
280 }
281}
282
283fn collect_ref_keys(value: &Value, refs: &mut BTreeSet<String>) {
284 match value {
285 Value::Array(items) => {
286 for item in items {
287 collect_ref_keys(item, refs);
288 }
289 }
290 Value::Object(record) => {
291 if let Some(reference) = record.get("$ref").and_then(Value::as_str)
292 && reference.starts_with("#/")
293 {
294 refs.insert(reference.to_string());
295 }
296
297 for nested in record.values() {
298 collect_ref_keys(nested, refs);
299 }
300 }
301 _ => {}
302 }
303}
304
305fn resolve_json_pointer<'a>(root: &'a Value, pointer: &str) -> Option<&'a Value> {
306 if !pointer.starts_with("#/") {
307 return None;
308 }
309
310 let mut current = root;
311 for part in pointer[2..].split('/') {
312 let decoded = part.replace("~1", "/").replace("~0", "~");
313 let next = current.as_object()?.get(&decoded)?;
314 current = next;
315 }
316
317 Some(current)
318}
319
320fn pick_schema_from_content(content: Option<&Value>) -> Option<Value> {
321 let content_record = as_object(content?)?;
322
323 let mut keys: Vec<String> = content_record.keys().cloned().collect();
324 keys.sort();
325
326 let mut preferred = Vec::with_capacity(keys.len() + 1);
327 preferred.push("application/json".to_string());
328 preferred.extend(keys);
329
330 let mut seen = HashSet::new();
331 for media_type in preferred {
332 if !seen.insert(media_type.clone()) {
333 continue;
334 }
335
336 let Some(media_type_value) = content_record.get(&media_type).and_then(as_object) else {
337 continue;
338 };
339
340 if let Some(schema) = media_type_value.get("schema") {
341 return Some(schema.clone());
342 }
343 }
344
345 None
346}
347
348fn extract_request_body_schema(operation: &Map<String, Value>) -> Option<Value> {
349 let request_body = as_object(operation.get("requestBody")?)?;
350 pick_schema_from_content(request_body.get("content"))
351}
352
353fn extract_response_schema(operation: &Map<String, Value>) -> Option<Value> {
354 let responses = as_object(operation.get("responses")?)?;
355 let mut response_codes: Vec<String> = responses.keys().cloned().collect();
356 response_codes.sort_by(|left, right| {
357 response_status_rank(left)
358 .cmp(&response_status_rank(right))
359 .then_with(|| left.cmp(right))
360 });
361
362 for response_code in response_codes {
363 let Some(response) = responses.get(&response_code).and_then(as_object) else {
364 continue;
365 };
366
367 let schema = pick_schema_from_content(response.get("content"));
368 if schema.is_some() {
369 return schema;
370 }
371 }
372
373 None
374}
375
376fn collect_parameter_schema_by_key(
377 path_item: &Map<String, Value>,
378 operation: &Map<String, Value>,
379) -> HashMap<String, Value> {
380 let mut schemas = HashMap::new();
381
382 let mut add_parameters = |candidate: Option<&Value>| {
383 let Some(items) = candidate.and_then(Value::as_array) else {
384 return;
385 };
386
387 for item in items {
388 let Some(item_record) = as_object(item) else {
389 continue;
390 };
391
392 let Some(parameter) = to_extracted_tool_parameter(item) else {
393 continue;
394 };
395
396 let Some(schema) = item_record.get("schema") else {
397 continue;
398 };
399
400 schemas.insert(
401 format!("{}:{}", parameter.location, parameter.name),
402 schema.clone(),
403 );
404 }
405 };
406
407 add_parameters(path_item.get("parameters"));
408 add_parameters(operation.get("parameters"));
409
410 schemas
411}
412
413fn build_input_schema(
414 path_item: &Map<String, Value>,
415 operation: &Map<String, Value>,
416 invocation: &OpenApiInvocationPayload,
417) -> Option<Value> {
418 let parameter_schema_by_key = collect_parameter_schema_by_key(path_item, operation);
419
420 let mut properties = Map::new();
421 let mut required = BTreeSet::new();
422
423 for parameter in &invocation.parameters {
424 let key = format!("{}:{}", parameter.location, parameter.name);
425 let schema = parameter_schema_by_key
426 .get(&key)
427 .cloned()
428 .unwrap_or_else(|| json!({ "type": "string" }));
429
430 properties.insert(parameter.name.clone(), schema);
431
432 if parameter.required {
433 required.insert(parameter.name.clone());
434 }
435 }
436
437 let request_body_schema = extract_request_body_schema(operation);
438 if let Some(schema) = request_body_schema {
439 properties.insert("body".to_string(), schema);
440
441 if invocation
442 .request_body
443 .as_ref()
444 .map(|request_body| request_body.required)
445 == Some(true)
446 {
447 required.insert("body".to_string());
448 }
449 }
450
451 if properties.is_empty() {
452 return None;
453 }
454
455 let required_list: Vec<String> = required.into_iter().collect();
456
457 Some(json!({
458 "type": "object",
459 "properties": properties,
460 "required": required_list,
461 "additionalProperties": false,
462 }))
463}
464
465fn build_tool_typing(
466 path_item: &Map<String, Value>,
467 operation: &Map<String, Value>,
468 invocation: &OpenApiInvocationPayload,
469) -> Result<Option<DiscoveryTypingPayload>, String> {
470 let input_schema = build_input_schema(path_item, operation, invocation);
471 let output_schema = extract_response_schema(operation);
472
473 if input_schema.is_none() && output_schema.is_none() {
474 return Ok(None);
475 }
476
477 let mut refs = BTreeSet::new();
478 if let Some(schema) = &input_schema {
479 collect_ref_keys(schema, &mut refs);
480 }
481
482 if let Some(schema) = &output_schema {
483 collect_ref_keys(schema, &mut refs);
484 }
485
486 let ref_hint_keys = if refs.is_empty() {
487 None
488 } else {
489 Some(refs.into_iter().collect())
490 };
491
492 Ok(Some(DiscoveryTypingPayload {
493 input_schema_json: input_schema
494 .as_ref()
495 .map(encode_stable_json)
496 .transpose()?,
497 output_schema_json: output_schema
498 .as_ref()
499 .map(encode_stable_json)
500 .transpose()?,
501 ref_hint_keys,
502 }))
503}
504
505fn build_ref_hint_table(open_api_spec: &Value, initial_ref_keys: &[String]) -> Result<BTreeMap<String, String>, String> {
506 let mut queue = VecDeque::new();
507 let mut queued = HashSet::new();
508
509 for ref_key in initial_ref_keys {
510 if queued.insert(ref_key.clone()) {
511 queue.push_back(ref_key.clone());
512 }
513 }
514
515 let mut seen = HashSet::new();
516 let mut table = BTreeMap::new();
517
518 while let Some(ref_key) = queue.pop_front() {
519 if !seen.insert(ref_key.clone()) {
520 continue;
521 }
522
523 let Some(resolved) = resolve_json_pointer(open_api_spec, &ref_key) else {
524 continue;
525 };
526
527 table.insert(ref_key.clone(), encode_stable_json(resolved)?);
528
529 let mut nested = BTreeSet::new();
530 collect_ref_keys(resolved, &mut nested);
531 for nested_ref in nested {
532 if !seen.contains(&nested_ref) {
533 queue.push_back(nested_ref);
534 }
535 }
536 }
537
538 Ok(table)
539}
540
541fn normalize_path_for_tool_id(path_value: &str) -> String {
542 let trimmed = path_value.trim();
543 let no_leading = trimmed.strip_prefix('/').unwrap_or(trimmed);
544
545 let mut normalized = String::new();
546 let mut last_was_underscore = false;
547
548 for character in no_leading.chars() {
549 if character.is_ascii_alphanumeric() {
550 normalized.push(character);
551 last_was_underscore = false;
552 continue;
553 }
554
555 if !last_was_underscore {
556 normalized.push('_');
557 last_was_underscore = true;
558 }
559 }
560
561 let without_edges = normalized.trim_matches('_').to_lowercase();
562 if without_edges.is_empty() {
563 return "root".to_string();
564 }
565
566 without_edges
567}
568
569fn build_tool_id(method: &str, path_value: &str, operation: &Map<String, Value>) -> String {
570 if let Some(operation_id) = as_trimmed_string(operation.get("operationId")) {
571 return operation_id;
572 }
573
574 format!("{}_{}", method, normalize_path_for_tool_id(path_value))
575}
576
577fn build_tool_name(method: &str, path_value: &str, operation: &Map<String, Value>) -> String {
578 if let Some(summary) = as_trimmed_string(operation.get("summary")) {
579 return summary;
580 }
581
582 if let Some(operation_id) = as_trimmed_string(operation.get("operationId")) {
583 return operation_id;
584 }
585
586 format!("{} {}", method.to_uppercase(), path_value)
587}
588
589fn build_tool_description(operation: &Map<String, Value>) -> Option<String> {
590 as_trimmed_string(operation.get("description")).or_else(|| as_trimmed_string(operation.get("summary")))
591}
592
593fn ensure_unique_tool_ids(source_name: &str, tools: &[OpenApiExtractedTool]) -> Result<(), String> {
594 let mut seen = HashSet::new();
595
596 for tool in tools {
597 if !seen.insert(tool.tool_id.clone()) {
598 return Err(format!(
599 "OpenAPI extraction failed for '{source_name}': duplicate toolId {} ({})",
600 tool.tool_id,
601 tool.path
602 ));
603 }
604 }
605
606 Ok(())
607}
608
609fn extract_openapi_manifest(source_name: &str, openapi_spec: &Value) -> Result<OpenApiToolManifest, String> {
610 let spec_record = as_object(openapi_spec)
611 .ok_or_else(|| format!("OpenAPI extraction failed for '{source_name}': spec must be object"))?;
612
613 let Some(paths) = spec_record.get("paths").and_then(as_object) else {
614 return Ok(OpenApiToolManifest {
615 version: 1,
616 source_hash: hash_unknown(openapi_spec)?,
617 tools: Vec::new(),
618 ref_hint_table: None,
619 });
620 };
621
622 let mut path_keys: Vec<String> = paths.keys().cloned().collect();
623 path_keys.sort();
624
625 let mut tools = Vec::new();
626
627 for path_value in path_keys {
628 let Some(path_item) = paths.get(&path_value).and_then(as_object) else {
629 continue;
630 };
631
632 for method in OPEN_API_HTTP_METHODS {
633 let Some(operation) = path_item.get(method).and_then(as_object) else {
634 continue;
635 };
636
637 let invocation = build_invocation_metadata(method, &path_value, path_item, operation);
638 let typing = build_tool_typing(path_item, operation, &invocation)?;
639
640 let operation_hash_input = json!({
641 "method": method,
642 "path": path_value,
643 "operation": operation,
644 "invocation": invocation,
645 });
646
647 tools.push(OpenApiExtractedTool {
648 tool_id: build_tool_id(method, &path_value, operation),
649 name: build_tool_name(method, &path_value, operation),
650 description: build_tool_description(operation),
651 method: method.to_string(),
652 path: path_value.clone(),
653 invocation,
654 operation_hash: hash_unknown(&operation_hash_input)?,
655 typing,
656 });
657 }
658 }
659
660 tools.sort_by(|left, right| {
661 left.tool_id
662 .to_lowercase()
663 .cmp(&right.tool_id.to_lowercase())
664 .then_with(|| left.tool_id.cmp(&right.tool_id))
665 });
666 ensure_unique_tool_ids(source_name, &tools)?;
667
668 let direct_ref_keys: Vec<String> = tools
669 .iter()
670 .flat_map(|tool| {
671 tool.typing
672 .as_ref()
673 .and_then(|typing| typing.ref_hint_keys.clone())
674 .unwrap_or_default()
675 })
676 .collect();
677
678 let ref_hint_table = build_ref_hint_table(openapi_spec, &direct_ref_keys)?;
679
680 Ok(OpenApiToolManifest {
681 version: 1,
682 source_hash: hash_unknown(openapi_spec)?,
683 tools,
684 ref_hint_table: if ref_hint_table.is_empty() {
685 None
686 } else {
687 Some(ref_hint_table)
688 },
689 })
690}
tools/openapi-extractor-rs/src/main.rs+58−0
@@ -0,0 +1,58 @@
1use std::fs;
2use std::io::{self, Read, Write};
3
4use clap::Parser;
5
6#[derive(Parser, Debug)]
7#[command(name = "openapi-extractor-rs")]
8struct Cli {
9 #[arg(long)]
10 source_name: String,
11 #[arg(long)]
12 input: String,
13 #[arg(long)]
14 output: Option<String>,
15 #[arg(long, default_value_t = false)]
16 pretty: bool,
17}
18
19fn main() {
20 if let Err(error) = run() {
21 eprintln!("{error}");
22 std::process::exit(1);
23 }
24}
25
26fn run() -> Result<(), String> {
27 let args = Cli::parse();
28 let source_text = read_input_text(&args.input)?;
29
30 let output_json =
31 openapi_extractor_rs::extract_manifest_json(&args.source_name, &source_text, args.pretty)?;
32
33 if let Some(path) = args.output {
34 fs::write(path, output_json).map_err(|error| format!("write output file: {error}"))?;
35 } else {
36 let mut stdout = io::stdout().lock();
37 stdout
38 .write_all(output_json.as_bytes())
39 .map_err(|error| format!("write stdout: {error}"))?;
40 stdout
41 .write_all(b"\n")
42 .map_err(|error| format!("write stdout newline: {error}"))?;
43 }
44
45 Ok(())
46}
47
48fn read_input_text(input: &str) -> Result<String, String> {
49 if input == "-" {
50 let mut buffer = String::new();
51 io::stdin()
52 .read_to_string(&mut buffer)
53 .map_err(|error| format!("read stdin: {error}"))?;
54 return Ok(buffer);
55 }
56
57 fs::read_to_string(input).map_err(|error| format!("read input file '{input}': {error}"))
58}