History

Add research page on MCP tool presentation across harnesses

Rhys Sullivan committed 2 months ago

Commit 8aaf41e1 parent 4c9e7551 file · +598 −0Browse files

Unified · Split

apps/marketing/src/pages/research/mcp-tool-visibility.astro+598−0
@@ -0,0 +1,598 @@
1---
2import Layout from "../../layouts/Layout.astro";
3
4const GH = "https://github.com/UsefulSoftwareCo/executor";
5const title = "How agents actually see your MCP tools";
6const description =
7 "The same MCP server was connected to five coding agents and the model-bound traffic was measured. Each harness shows the model something different: full definitions, bare names, or nothing at all.";
8
9// Δ prompt tokens for the 31 per-integration search tools, provider-reported.
10const costs = [
11 { name: "Claude Code", detail: "2.1.247 · inline", delta: 3464, plain: "36,013", search: "39,477" },
12 { name: "opencode (beta)", detail: "default · inline", delta: 3400, plain: "22,118", search: "25,518" },
13 { name: "opencode", detail: "1.18.15 · inline", delta: 3399, plain: "17,458", search: "20,857" },
14 { name: "opencode (beta)", detail: "code mode · TS signatures", delta: 1280, plain: "16,126", search: "17,406" },
15 { name: "Cursor CLI", detail: "2026.07.23 · names only", delta: 170, plain: "16,140", search: "16,310" },
16 { name: "Codex CLI", detail: "0.147.0 · code mode", delta: 0, plain: "15,776", search: "15,776" },
17];
18const max = Math.max(...costs.map((c) => c.delta));
19---
20
21<Layout title={`${title} — Executor`} description={description}>
22 <main class="bg-soft relative min-h-dvh antialiased">
23 <div
24 class="pattern-bg pattern-graph"
25 style="position:absolute;top:0;left:0;right:0;height:60vh;z-index:1;pointer-events:none;"
26 >
27 </div>
28 <div
29 style="position:absolute;top:0;left:0;right:0;height:60vh;z-index:2;pointer-events:none;background:linear-gradient(180deg,transparent 0%,transparent 55%,var(--color-surface) 100%);"
30 >
31 </div>
32
33 <div class="relative z-10">
34 <nav>
35 <div class="mx-auto flex h-14 max-w-[1200px] items-center justify-between px-6 sm:px-10">
36 <a href="/" class="flex items-center gap-2.5">
37 <img src="/favicon-192.png" alt="" class="h-5 w-5" />
38 <span class="text-[15px] font-semibold tracking-[-0.01em] text-ink">executor</span>
39 </a>
40 <div class="flex items-center gap-5">
41 <a href="/blog" class="text-[14px] font-medium text-ink-2 transition-colors hover:text-ink">Blog</a>
42 <a
43 href="https://executor.sh/docs"
44 class="hidden text-[14px] font-medium text-ink-2 transition-colors hover:text-ink sm:inline">Docs</a
45 >
46 <a href={GH} class="text-[14px] font-medium text-ink-2 transition-colors hover:text-ink">GitHub</a>
47 <a href="/#install" class="btn-primary">Get started →</a>
48 </div>
49 </div>
50 </nav>
51
52 {/* ─── HEADER ─── */}
53 <section class="px-6 pt-16 pb-10 sm:px-10 sm:pt-24 sm:pb-14">
54 <div class="mx-auto max-w-[820px]">
55 <span class="eyebrow">Research · August 2026</span>
56 <h1
57 class="mt-5 text-[clamp(2.2rem,5vw,3.6rem)] font-semibold leading-[1.05] tracking-[-0.025em] text-ink text-balance"
58 >
59 {title}
60 </h1>
61 <p class="mt-5 max-w-[62ch] text-[17px] leading-[1.6] text-ink-2 text-pretty">
62 {description} We connected one Executor workspace — 31 integrations — to Claude Code,
63 Cursor's CLI, Codex, and two versions of opencode, twice each: once plain, once with
64 Executor's <code class="mono-inline">?search_tools=true</code> option, which adds one
65 minimal <code class="mono-inline">search_&lt;integration&gt;</code> tool per connected
66 integration. The difference between those two sessions is exactly what 31 extra tool
67 definitions cost, and where they end up.
68 </p>
69 </div>
70 </section>
71
72 {/* ─── THE THREE REALITIES ─── */}
73 <section class="border-t border-rule px-6 py-14 sm:px-10 sm:py-20">
74 <div class="mx-auto max-w-[1100px]">
75 <span class="eyebrow">What each harness shows the model</span>
76 <h2 class="section-title mt-4 max-w-[24ch]">Five harnesses, three different realities</h2>
77 <p class="section-sub mt-4 max-w-[62ch]">
78 "The model sees your tools" hides three different mechanisms. Every card below is one
79 harness's view of the same server, drawn from the captured request that actually went
80 to the model. Solid blocks are in the context window on every turn; dashed blocks are
81 fetched on demand; crossed-out blocks never reach the model at all.
82 </p>
83
84 <div class="mt-10 grid gap-6">
85 {/* Inliners */}
86 <div class="persp surface-card">
87 <div class="persp-head">
88 <div>
89 <span class="persp-name">Claude Code · opencode</span>
90 <span class="persp-family">family: inline — every definition, every turn</span>
91 </div>
92 <span class="persp-cost">+3,399–3,464 tokens</span>
93 </div>
94 <div class="persp-body">
95 <div class="ctx">
96 <div class="ctx-label">the model's context window</div>
97 <div class="ctx-block ctx-quiet">system prompt</div>
98 <div class="ctx-block ctx-loud">
99 <b>38 tool definitions — full JSON schema each</b>
100 <span>name + description + parameters for every tool, inlined as API function tools</span>
101 </div>
102 <div class="ctx-block ctx-quiet">conversation</div>
103 </div>
104 <div class="persp-detail">
105 <div class="code-window">
106 <div class="code-titlebar">
107 <span class="code-dot"></span><span class="code-dot"></span><span class="code-dot"></span>
108 <span class="code-filename">tools[7] of 38 — as sent to the model</span>
109 </div>
110 <pre
111 class="code-body">{`{
112 "name": "mcp__executor__search_mcp_linear_app",
113 "description": "Search this integration's tools;
114 empty query lists all. Run results with execute.",
115 "input_schema": {
116 "type": "object",
117 "properties": { "query": { "type": "string" } }
118 }
119}`}</pre>
120 </div>
121 <p class="persp-path">
122 <span class="persp-path-label">To search Linear’s tools</span>
123 one call — <code class="mono-inline">search_mcp_linear_app({"{ query }"})</code>,
124 the definition is already in context.
125 </p>
126 </div>
127 </div>
128 </div>
129
130 {/* opencode code mode */}
131 <div class="persp surface-card">
132 <div class="persp-head">
133 <div>
134 <span class="persp-name">opencode · code mode</span>
135 <span class="persp-family">family: code mode, transparent — tools become typed signatures</span>
136 </div>
137 <span class="persp-cost">+1,280 tokens</span>
138 </div>
139 <div class="persp-body">
140 <div class="ctx">
141 <div class="ctx-label">the model's context window</div>
142 <div class="ctx-block ctx-quiet">system prompt</div>
143 <div class="ctx-block ctx-quiet">13 built-in tools</div>
144 <div class="ctx-block ctx-loud">
145 <b>one <code class="mono-inline">execute</code> tool</b>
146 <span>its description lists every MCP tool as a TypeScript signature — 2.7× denser than JSON definitions</span>
147 </div>
148 <div class="ctx-block ctx-quiet">conversation</div>
149 </div>
150 <div class="persp-detail">
151 <div class="code-window">
152 <div class="code-titlebar">
153 <span class="code-dot"></span><span class="code-dot"></span><span class="code-dot"></span>
154 <span class="code-filename">inside execute's description</span>
155 </div>
156 <pre
157 class="code-body">{`- tools.executor.search_google_gmail(input: {
158 query?: string,
159 }): Promise<unknown> // Search this integration's
160 tools; empty query lists all. Run results
161 with execute.`}</pre>
162 </div>
163 <p class="persp-path">
164 <span class="persp-path-label">To search Linear’s tools</span>
165 one call through code —
166 <code class="mono-inline">execute("await tools.executor.search_mcp_linear_app(…)")</code>.
167 </p>
168 </div>
169 </div>
170 </div>
171
172 {/* Cursor */}
173 <div class="persp surface-card">
174 <div class="persp-head">
175 <div>
176 <span class="persp-name">Cursor CLI</span>
177 <span class="persp-family">family: virtualized — a names-only catalog</span>
178 </div>
179 <span class="persp-cost">+170 tokens</span>
180 </div>
181 <div class="persp-body">
182 <div class="ctx">
183 <div class="ctx-label">the model's context window</div>
184 <div class="ctx-block ctx-quiet">system prompt</div>
185 <div class="ctx-block ctx-quiet">19 first-party tools, incl. <code class="mono-inline">GetDynamicTools</code> / <code class="mono-inline">CallDynamicTool</code></div>
186 <div class="ctx-block ctx-loud">
187 <b>catalog: bare tool names, grouped by server</b>
188 <span>≈5.5 tokens per tool — the name is all the model gets</span>
189 </div>
190 <div class="ctx-block ctx-dashed">
191 descriptions &amp; schemas — fetched on demand via <code class="mono-inline">GetDynamicTools</code>
192 </div>
193 <div class="ctx-block ctx-quiet">conversation</div>
194 </div>
195 <div class="persp-detail">
196 <div class="code-window">
197 <div class="code-titlebar">
198 <span class="code-dot"></span><span class="code-dot"></span><span class="code-dot"></span>
199 <span class="code-filename">the model's catalog view</span>
200 </div>
201 <pre
202 class="code-body">{`executor: execute · skills · resume ·
203 search_axiom_mcp · search_github_rest ·
204 search_google_gmail · search_mcp_linear_app · …
205
206(full definition available via GetDynamicTools —
207 not fetched unless the model asks)`}</pre>
208 </div>
209 <p class="persp-path">
210 <span class="persp-path-label">To search Linear’s tools</span>
211 the name is visible; call it through
212 <code class="mono-inline">CallDynamicTool("search_mcp_linear_app", …)</code>,
213 optionally fetching the schema first.
214 </p>
215 </div>
216 </div>
217 </div>
218
219 {/* Codex */}
220 <div class="persp surface-card">
221 <div class="persp-head">
222 <div>
223 <span class="persp-name">Codex CLI</span>
224 <span class="persp-family">family: code mode, opaque — nothing reaches the prompt</span>
225 </div>
226 <span class="persp-cost">+0 tokens</span>
227 </div>
228 <div class="persp-body">
229 <div class="ctx">
230 <div class="ctx-label">the model's context window</div>
231 <div class="ctx-block ctx-quiet">system prompt</div>
232 <div class="ctx-block ctx-quiet">9 native tools — <code class="mono-inline">exec</code>, collaboration, …</div>
233 <div class="ctx-block ctx-never">
234 <b>your MCP tools</b>
235 <span>carried only in request metadata the model never reads; prompts were byte-identical with 7 vs 38 tools</span>
236 </div>
237 <div class="ctx-block ctx-quiet">conversation</div>
238 </div>
239 <div class="persp-detail">
240 <div class="code-window">
241 <div class="code-titlebar">
242 <span class="code-dot"></span><span class="code-dot"></span><span class="code-dot"></span>
243 <span class="code-filename">request metadata — not model-visible</span>
244 </div>
245 <pre
246 class="code-body">{`"code_mode_tool_names": {
247 "search_google_gmail":
248 { "namespace": "mcp__executor" },
249 … 37 more
250}`}</pre>
251 </div>
252 <p class="persp-path">
253 <span class="persp-path-label">To search Linear’s tools</span>
254 the model must call <code class="mono-inline">exec</code> and write code —
255 <code class="mono-inline">await tools.mcp__executor__search_mcp_linear_app(…)</code> —
256 discovering what exists only from inside the sandbox.
257 </p>
258 </div>
259 </div>
260 </div>
261 </div>
262 </div>
263 </section>
264
265 {/* ─── COST ─── */}
266 <section class="border-t border-rule px-6 py-14 sm:px-10 sm:py-20">
267 <div class="mx-auto max-w-[1100px]">
268 <span class="eyebrow">Measured, not estimated</span>
269 <h2 class="section-title mt-4 max-w-[26ch]">What 31 tool definitions cost, by harness</h2>
270 <p class="section-sub mt-4 max-w-[62ch]">
271 Δ prompt tokens between the plain and <code class="mono-inline">?search_tools=true</code> sessions,
272 as reported by the provider's usage accounting (input + cache write + cache read). Same
273 workspace, same prompt; only the endpoint query differs.
274 </p>
275 <div class="mt-10 surface-card cost-card">
276 {
277 costs.map((c) => (
278 <div class="cost-row">
279 <div class="cost-name">
280 {c.name}
281 <span class="cost-detail">{c.detail}</span>
282 </div>
283 <div class="cost-track" title={`plain ${c.plain} → search ${c.search}`}>
284 <div
285 class={`cost-bar${c.delta === 0 ? " cost-bar--zero" : ""}`}
286 style={`width:${(c.delta / max) * 100}%`}
287 />
288 </div>
289 <div class="cost-val">{c.delta === 0 ? "0" : `+${c.delta.toLocaleString()}`}</div>
290 </div>
291 ))
292 }
293 </div>
294 <p class="mt-4 text-[13px] leading-[1.6] text-ink-3">
295 Cursor's model traffic terminates at its backend, so its pair comes from Cursor's own
296 per-request usage meter rather than a captured prompt. JSON tool definitions tokenize at
297 roughly 2.4 characters per token — schema punctuation is token-dense.
298 </p>
299 </div>
300 </section>
301
302 {/* ─── TAKEAWAYS ─── */}
303 <section class="border-t border-rule px-6 py-14 sm:px-10 sm:py-20">
304 <div class="mx-auto max-w-[1100px]">
305 <span class="eyebrow">If you ship an MCP server</span>
306 <h2 class="section-title mt-4 max-w-[24ch]">Design for the weakest channel</h2>
307 <div class="cap-grid mt-10" style="grid-template-columns:repeat(auto-fit,minmax(240px,1fr));">
308 <div class="cap-card">
309 <span class="cap-num">01</span>
310 <h3>The name is the only universal channel</h3>
311 <p>
312 Every harness that shows tools at all shows the name. Descriptions reach only the
313 inliners; schemas effectively reach only the inliners. If information must arrive,
314 put it in the name.
315 </p>
316 </div>
317 <div class="cap-card">
318 <span class="cap-num">02</span>
319 <h3>Schema tricks don't travel</h3>
320 <p>
321 An enum of options inside a parameter looks clever and costs less — and is invisible
322 in Cursor and Codex, the two harnesses where you'd need it most.
323 </p>
324 </div>
325 <div class="cap-card">
326 <span class="cap-num">03</span>
327 <h3>Keep definitions small anyway</h3>
328 <p>
329 Where tools are inlined, every byte is paid on every turn, multiplied by tool count.
330 One shared description sentence and a single parameter cut this surface by more than half.
331 </p>
332 </div>
333 <div class="cap-card">
334 <span class="cap-num">04</span>
335 <h3>Measure with provider usage, not estimates</h3>
336 <p>
337 Client-side token meters and character counts both mislead. Run the same session
338 twice and diff what the provider bills — that number is the truth.
339 </p>
340 </div>
341 </div>
342 </div>
343 </section>
344
345 {/* ─── BOTTOM CTA ─── */}
346 <section class="border-t border-rule py-16 sm:py-20">
347 <div class="mx-auto max-w-[720px] px-6 text-center sm:px-10">
348 <h2 class="section-title mx-auto mb-7 max-w-[20ch]">
349 Connect any agent to <span class="serif-italic">everything</span>.
350 </h2>
351 <div class="flex flex-wrap items-center justify-center gap-3">
352 <a href="/#install" class="btn-primary btn-primary--hero">Get started →</a>
353 <a href={GH} class="btn-secondary">Star on GitHub</a>
354 </div>
355 </div>
356 </section>
357
358 {/* ─── FOOTER ─── */}
359 <footer class="border-t border-rule py-10">
360 <div
361 class="mx-auto flex max-w-[1200px] flex-col items-start justify-between gap-6 px-6 sm:flex-row sm:items-center sm:px-10"
362 >
363 <div class="flex items-center gap-2.5">
364 <img src="/favicon-192.png" alt="" class="h-4 w-4 opacity-80" />
365 <span class="text-[13px] font-medium text-ink-2">executor</span>
366 <span class="ml-2 text-[12px] text-ink-3">© {new Date().getFullYear()}</span>
367 </div>
368 <div class="flex items-center gap-6 text-[13px] text-ink-2">
369 <a href="/blog" class="transition-colors hover:text-ink">Blog</a>
370 <a href="https://executor.sh/docs" class="transition-colors hover:text-ink">Docs</a>
371 <a href={GH} class="transition-colors hover:text-ink">GitHub</a>
372 <a href="/privacy" class="transition-colors hover:text-ink">Privacy</a>
373 <a href="/terms" class="transition-colors hover:text-ink">Terms</a>
374 </div>
375 </div>
376 </footer>
377 </div>
378 </main>
379</Layout>
380
381<style>
382 .mono-inline {
383 font-family: var(--font-mono);
384 font-size: 0.88em;
385 background: var(--color-surface-2);
386 border: 1px solid var(--color-rule);
387 border-radius: 4px;
388 padding: 1px 5px;
389 white-space: nowrap;
390 }
391
392 /* ── perspective cards ── */
393 .persp {
394 padding: 0;
395 overflow: hidden;
396 }
397 .persp-head {
398 display: flex;
399 align-items: baseline;
400 justify-content: space-between;
401 gap: 16px;
402 padding: 18px 24px;
403 border-bottom: 1px solid var(--color-rule);
404 }
405 .persp-name {
406 font-weight: 600;
407 font-size: 17px;
408 letter-spacing: -0.01em;
409 color: var(--color-ink);
410 }
411 .persp-family {
412 display: block;
413 margin-top: 3px;
414 font-family: var(--font-mono);
415 font-size: 11px;
416 letter-spacing: 0.08em;
417 text-transform: uppercase;
418 color: var(--color-ink-3);
419 }
420 .persp-cost {
421 font-family: var(--font-mono);
422 font-size: 13px;
423 color: var(--color-ink);
424 white-space: nowrap;
425 }
426 .persp-body {
427 display: grid;
428 grid-template-columns: 340px 1fr;
429 gap: 0;
430 }
431 @media (max-width: 820px) {
432 .persp-body {
433 grid-template-columns: 1fr;
434 }
435 }
436
437 /* context-window schematic */
438 .ctx {
439 padding: 20px 24px;
440 border-right: 1px solid var(--color-rule);
441 display: flex;
442 flex-direction: column;
443 gap: 6px;
444 }
445 @media (max-width: 820px) {
446 .ctx {
447 border-right: 0;
448 border-bottom: 1px solid var(--color-rule);
449 }
450 }
451 .ctx-label {
452 font-family: var(--font-mono);
453 font-size: 10.5px;
454 letter-spacing: 0.08em;
455 text-transform: uppercase;
456 color: var(--color-ink-3);
457 margin-bottom: 4px;
458 }
459 .ctx-block {
460 border-radius: 6px;
461 padding: 9px 12px;
462 font-size: 12.5px;
463 line-height: 1.45;
464 }
465 .ctx-quiet {
466 border: 1px solid var(--color-rule);
467 background: var(--color-surface-2);
468 color: var(--color-ink-2);
469 }
470 .ctx-loud {
471 border: 1px solid var(--color-rule-strong);
472 background: var(--color-surface);
473 color: var(--color-ink);
474 }
475 .ctx-loud b {
476 display: block;
477 font-weight: 600;
478 font-size: 13px;
479 }
480 .ctx-loud span {
481 display: block;
482 margin-top: 2px;
483 color: var(--color-ink-2);
484 font-size: 12px;
485 }
486 .ctx-dashed {
487 border: 1px dashed var(--color-rule-strong);
488 color: var(--color-ink-3);
489 background: transparent;
490 }
491 .ctx-never {
492 border: 1px dashed var(--color-rule);
493 background: transparent;
494 color: var(--color-ink-3);
495 }
496 .ctx-never b {
497 display: block;
498 font-weight: 500;
499 font-size: 13px;
500 text-decoration: line-through;
501 color: var(--color-ink-3);
502 }
503 .ctx-never span {
504 display: block;
505 margin-top: 2px;
506 font-size: 12px;
507 }
508
509 .persp-detail {
510 padding: 20px 24px;
511 display: flex;
512 flex-direction: column;
513 gap: 14px;
514 min-width: 0;
515 }
516 .persp-detail .code-window {
517 margin: 0;
518 }
519 .persp-detail .code-body {
520 font-size: 12px;
521 line-height: 1.6;
522 overflow-x: auto;
523 }
524 .persp-path {
525 font-size: 13.5px;
526 line-height: 1.6;
527 color: var(--color-ink-2);
528 }
529 .persp-path-label {
530 display: block;
531 font-family: var(--font-mono);
532 font-size: 10.5px;
533 letter-spacing: 0.08em;
534 text-transform: uppercase;
535 color: var(--color-ink-3);
536 margin-bottom: 3px;
537 }
538
539 /* ── cost chart ── */
540 .cost-card {
541 padding: 10px 24px;
542 }
543 .cost-row {
544 display: grid;
545 grid-template-columns: 240px 1fr 84px;
546 align-items: center;
547 gap: 16px;
548 padding: 11px 0;
549 }
550 .cost-row + .cost-row {
551 border-top: 1px solid var(--color-rule);
552 }
553 @media (max-width: 640px) {
554 .cost-row {
555 grid-template-columns: 1fr 84px;
556 }
557 .cost-track {
558 display: none;
559 }
560 }
561 .cost-name {
562 font-size: 14px;
563 font-weight: 500;
564 color: var(--color-ink);
565 }
566 .cost-detail {
567 display: block;
568 font-family: var(--font-mono);
569 font-size: 11px;
570 color: var(--color-ink-3);
571 margin-top: 2px;
572 }
573 .cost-track {
574 position: relative;
575 height: 20px;
576 }
577 .cost-bar {
578 position: absolute;
579 top: 3px;
580 bottom: 3px;
581 left: 0;
582 background: var(--color-ink);
583 border-radius: 0 4px 4px 0;
584 min-width: 3px;
585 }
586 .cost-bar--zero {
587 background: transparent;
588 border-left: 2px solid var(--color-ink);
589 border-radius: 0;
590 min-width: 0;
591 }
592 .cost-val {
593 font-family: var(--font-mono);
594 font-size: 13px;
595 text-align: right;
596 color: var(--color-ink);
597 }
598</style>

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t