History

Add research page on MCP tool presentation across harnesses

Rhys Sullivan committed 2 months ago

Commit 8aaf41e1 parent 4c9e7551 file · +598 −0Browse files

Unified · Split

apps/marketing/src/pages/research/mcp-tool-visibility.astro+598−0
@@ -0,0 +1,598 @@
1+---
2+import Layout from "../../layouts/Layout.astro";
3+
4+const GH = "https://github.com/UsefulSoftwareCo/executor";
5+const title = "How agents actually see your MCP tools";
6+const description =
7+ "The same MCP server was connected to five coding agents and the model-bound traffic was measured. Each harness shows the model something different: full definitions, bare names, or nothing at all.";
8+
9+// Δ prompt tokens for the 31 per-integration search tools, provider-reported.
10+const costs = [
11+ { name: "Claude Code", detail: "2.1.247 · inline", delta: 3464, plain: "36,013", search: "39,477" },
12+ { name: "opencode (beta)", detail: "default · inline", delta: 3400, plain: "22,118", search: "25,518" },
13+ { name: "opencode", detail: "1.18.15 · inline", delta: 3399, plain: "17,458", search: "20,857" },
14+ { name: "opencode (beta)", detail: "code mode · TS signatures", delta: 1280, plain: "16,126", search: "17,406" },
15+ { name: "Cursor CLI", detail: "2026.07.23 · names only", delta: 170, plain: "16,140", search: "16,310" },
16+ { name: "Codex CLI", detail: "0.147.0 · code mode", delta: 0, plain: "15,776", search: "15,776" },
17+];
18+const max = Math.max(...costs.map((c) => c.delta));
19+---
20+
21+<Layout title={`${title} — Executor`} description={description}>
22+ <main class="bg-soft relative min-h-dvh antialiased">
23+ <div
24+ class="pattern-bg pattern-graph"
25+ style="position:absolute;top:0;left:0;right:0;height:60vh;z-index:1;pointer-events:none;"
26+ >
27+ </div>
28+ <div
29+ style="position:absolute;top:0;left:0;right:0;height:60vh;z-index:2;pointer-events:none;background:linear-gradient(180deg,transparent 0%,transparent 55%,var(--color-surface) 100%);"
30+ >
31+ </div>
32+
33+ <div class="relative z-10">
34+ <nav>
35+ <div class="mx-auto flex h-14 max-w-[1200px] items-center justify-between px-6 sm:px-10">
36+ <a href="/" class="flex items-center gap-2.5">
37+ <img src="/favicon-192.png" alt="" class="h-5 w-5" />
38+ <span class="text-[15px] font-semibold tracking-[-0.01em] text-ink">executor</span>
39+ </a>
40+ <div class="flex items-center gap-5">
41+ <a href="/blog" class="text-[14px] font-medium text-ink-2 transition-colors hover:text-ink">Blog</a>
42+ <a
43+ href="https://executor.sh/docs"
44+ class="hidden text-[14px] font-medium text-ink-2 transition-colors hover:text-ink sm:inline">Docs</a
45+ >
46+ <a href={GH} class="text-[14px] font-medium text-ink-2 transition-colors hover:text-ink">GitHub</a>
47+ <a href="/#install" class="btn-primary">Get started →</a>
48+ </div>
49+ </div>
50+ </nav>
51+
52+ {/* ─── HEADER ─── */}
53+ <section class="px-6 pt-16 pb-10 sm:px-10 sm:pt-24 sm:pb-14">
54+ <div class="mx-auto max-w-[820px]">
55+ <span class="eyebrow">Research · August 2026</span>
56+ <h1
57+ class="mt-5 text-[clamp(2.2rem,5vw,3.6rem)] font-semibold leading-[1.05] tracking-[-0.025em] text-ink text-balance"
58+ >
59+ {title}
60+ </h1>
61+ <p class="mt-5 max-w-[62ch] text-[17px] leading-[1.6] text-ink-2 text-pretty">
62+ {description} We connected one Executor workspace — 31 integrations — to Claude Code,
63+ Cursor's CLI, Codex, and two versions of opencode, twice each: once plain, once with
64+ Executor's <code class="mono-inline">?search_tools=true</code> option, which adds one
65+ minimal <code class="mono-inline">search_&lt;integration&gt;</code> tool per connected
66+ integration. The difference between those two sessions is exactly what 31 extra tool
67+ definitions cost, and where they end up.
68+ </p>
69+ </div>
70+ </section>
71+
72+ {/* ─── THE THREE REALITIES ─── */}
73+ <section class="border-t border-rule px-6 py-14 sm:px-10 sm:py-20">
74+ <div class="mx-auto max-w-[1100px]">
75+ <span class="eyebrow">What each harness shows the model</span>
76+ <h2 class="section-title mt-4 max-w-[24ch]">Five harnesses, three different realities</h2>
77+ <p class="section-sub mt-4 max-w-[62ch]">
78+ "The model sees your tools" hides three different mechanisms. Every card below is one
79+ harness's view of the same server, drawn from the captured request that actually went
80+ to the model. Solid blocks are in the context window on every turn; dashed blocks are
81+ fetched on demand; crossed-out blocks never reach the model at all.
82+ </p>
83+
84+ <div class="mt-10 grid gap-6">
85+ {/* Inliners */}
86+ <div class="persp surface-card">
87+ <div class="persp-head">
88+ <div>
89+ <span class="persp-name">Claude Code · opencode</span>
90+ <span class="persp-family">family: inline — every definition, every turn</span>
91+ </div>
92+ <span class="persp-cost">+3,399–3,464 tokens</span>
93+ </div>
94+ <div class="persp-body">
95+ <div class="ctx">
96+ <div class="ctx-label">the model's context window</div>
97+ <div class="ctx-block ctx-quiet">system prompt</div>
98+ <div class="ctx-block ctx-loud">
99+ <b>38 tool definitions — full JSON schema each</b>
100+ <span>name + description + parameters for every tool, inlined as API function tools</span>
101+ </div>
102+ <div class="ctx-block ctx-quiet">conversation</div>
103+ </div>
104+ <div class="persp-detail">
105+ <div class="code-window">
106+ <div class="code-titlebar">
107+ <span class="code-dot"></span><span class="code-dot"></span><span class="code-dot"></span>
108+ <span class="code-filename">tools[7] of 38 — as sent to the model</span>
109+ </div>
110+ <pre
111+ class="code-body">{`{
112+ "name": "mcp__executor__search_mcp_linear_app",
113+ "description": "Search this integration's tools;
114+ empty query lists all. Run results with execute.",
115+ "input_schema": {
116+ "type": "object",
117+ "properties": { "query": { "type": "string" } }
118+ }
119+}`}</pre>
120+ </div>
121+ <p class="persp-path">
122+ <span class="persp-path-label">To search Linear’s tools</span>
123+ one call — <code class="mono-inline">search_mcp_linear_app({"{ query }"})</code>,
124+ the definition is already in context.
125+ </p>
126+ </div>
127+ </div>
128+ </div>
129+
130+ {/* opencode code mode */}
131+ <div class="persp surface-card">
132+ <div class="persp-head">
133+ <div>
134+ <span class="persp-name">opencode · code mode</span>
135+ <span class="persp-family">family: code mode, transparent — tools become typed signatures</span>
136+ </div>
137+ <span class="persp-cost">+1,280 tokens</span>
138+ </div>
139+ <div class="persp-body">
140+ <div class="ctx">
141+ <div class="ctx-label">the model's context window</div>
142+ <div class="ctx-block ctx-quiet">system prompt</div>
143+ <div class="ctx-block ctx-quiet">13 built-in tools</div>
144+ <div class="ctx-block ctx-loud">
145+ <b>one <code class="mono-inline">execute</code> tool</b>
146+ <span>its description lists every MCP tool as a TypeScript signature — 2.7× denser than JSON definitions</span>
147+ </div>
148+ <div class="ctx-block ctx-quiet">conversation</div>
149+ </div>
150+ <div class="persp-detail">
151+ <div class="code-window">
152+ <div class="code-titlebar">
153+ <span class="code-dot"></span><span class="code-dot"></span><span class="code-dot"></span>
154+ <span class="code-filename">inside execute's description</span>
155+ </div>
156+ <pre
157+ class="code-body">{`- tools.executor.search_google_gmail(input: {
158+ query?: string,
159+ }): Promise<unknown> // Search this integration's
160+ tools; empty query lists all. Run results
161+ with execute.`}</pre>
162+ </div>
163+ <p class="persp-path">
164+ <span class="persp-path-label">To search Linear’s tools</span>
165+ one call through code —
166+ <code class="mono-inline">execute("await tools.executor.search_mcp_linear_app(…)")</code>.
167+ </p>
168+ </div>
169+ </div>
170+ </div>
171+
172+ {/* Cursor */}
173+ <div class="persp surface-card">
174+ <div class="persp-head">
175+ <div>
176+ <span class="persp-name">Cursor CLI</span>
177+ <span class="persp-family">family: virtualized — a names-only catalog</span>
178+ </div>
179+ <span class="persp-cost">+170 tokens</span>
180+ </div>
181+ <div class="persp-body">
182+ <div class="ctx">
183+ <div class="ctx-label">the model's context window</div>
184+ <div class="ctx-block ctx-quiet">system prompt</div>
185+ <div class="ctx-block ctx-quiet">19 first-party tools, incl. <code class="mono-inline">GetDynamicTools</code> / <code class="mono-inline">CallDynamicTool</code></div>
186+ <div class="ctx-block ctx-loud">
187+ <b>catalog: bare tool names, grouped by server</b>
188+ <span>≈5.5 tokens per tool — the name is all the model gets</span>
189+ </div>
190+ <div class="ctx-block ctx-dashed">
191+ descriptions &amp; schemas — fetched on demand via <code class="mono-inline">GetDynamicTools</code>
192+ </div>
193+ <div class="ctx-block ctx-quiet">conversation</div>
194+ </div>
195+ <div class="persp-detail">
196+ <div class="code-window">
197+ <div class="code-titlebar">
198+ <span class="code-dot"></span><span class="code-dot"></span><span class="code-dot"></span>
199+ <span class="code-filename">the model's catalog view</span>
200+ </div>
201+ <pre
202+ class="code-body">{`executor: execute · skills · resume ·
203+ search_axiom_mcp · search_github_rest ·
204+ search_google_gmail · search_mcp_linear_app · …
205+
206+(full definition available via GetDynamicTools —
207+ not fetched unless the model asks)`}</pre>
208+ </div>
209+ <p class="persp-path">
210+ <span class="persp-path-label">To search Linear’s tools</span>
211+ the name is visible; call it through
212+ <code class="mono-inline">CallDynamicTool("search_mcp_linear_app", …)</code>,
213+ optionally fetching the schema first.
214+ </p>
215+ </div>
216+ </div>
217+ </div>
218+
219+ {/* Codex */}
220+ <div class="persp surface-card">
221+ <div class="persp-head">
222+ <div>
223+ <span class="persp-name">Codex CLI</span>
224+ <span class="persp-family">family: code mode, opaque — nothing reaches the prompt</span>
225+ </div>
226+ <span class="persp-cost">+0 tokens</span>
227+ </div>
228+ <div class="persp-body">
229+ <div class="ctx">
230+ <div class="ctx-label">the model's context window</div>
231+ <div class="ctx-block ctx-quiet">system prompt</div>
232+ <div class="ctx-block ctx-quiet">9 native tools — <code class="mono-inline">exec</code>, collaboration, …</div>
233+ <div class="ctx-block ctx-never">
234+ <b>your MCP tools</b>
235+ <span>carried only in request metadata the model never reads; prompts were byte-identical with 7 vs 38 tools</span>
236+ </div>
237+ <div class="ctx-block ctx-quiet">conversation</div>
238+ </div>
239+ <div class="persp-detail">
240+ <div class="code-window">
241+ <div class="code-titlebar">
242+ <span class="code-dot"></span><span class="code-dot"></span><span class="code-dot"></span>
243+ <span class="code-filename">request metadata — not model-visible</span>
244+ </div>
245+ <pre
246+ class="code-body">{`"code_mode_tool_names": {
247+ "search_google_gmail":
248+ { "namespace": "mcp__executor" },
249+ … 37 more
250+}`}</pre>
251+ </div>
252+ <p class="persp-path">
253+ <span class="persp-path-label">To search Linear’s tools</span>
254+ the model must call <code class="mono-inline">exec</code> and write code —
255+ <code class="mono-inline">await tools.mcp__executor__search_mcp_linear_app(…)</code> —
256+ discovering what exists only from inside the sandbox.
257+ </p>
258+ </div>
259+ </div>
260+ </div>
261+ </div>
262+ </div>
263+ </section>
264+
265+ {/* ─── COST ─── */}
266+ <section class="border-t border-rule px-6 py-14 sm:px-10 sm:py-20">
267+ <div class="mx-auto max-w-[1100px]">
268+ <span class="eyebrow">Measured, not estimated</span>
269+ <h2 class="section-title mt-4 max-w-[26ch]">What 31 tool definitions cost, by harness</h2>
270+ <p class="section-sub mt-4 max-w-[62ch]">
271+ Δ prompt tokens between the plain and <code class="mono-inline">?search_tools=true</code> sessions,
272+ as reported by the provider's usage accounting (input + cache write + cache read). Same
273+ workspace, same prompt; only the endpoint query differs.
274+ </p>
275+ <div class="mt-10 surface-card cost-card">
276+ {
277+ costs.map((c) => (
278+ <div class="cost-row">
279+ <div class="cost-name">
280+ {c.name}
281+ <span class="cost-detail">{c.detail}</span>
282+ </div>
283+ <div class="cost-track" title={`plain ${c.plain} → search ${c.search}`}>
284+ <div
285+ class={`cost-bar${c.delta === 0 ? " cost-bar--zero" : ""}`}
286+ style={`width:${(c.delta / max) * 100}%`}
287+ />
288+ </div>
289+ <div class="cost-val">{c.delta === 0 ? "0" : `+${c.delta.toLocaleString()}`}</div>
290+ </div>
291+ ))
292+ }
293+ </div>
294+ <p class="mt-4 text-[13px] leading-[1.6] text-ink-3">
295+ Cursor's model traffic terminates at its backend, so its pair comes from Cursor's own
296+ per-request usage meter rather than a captured prompt. JSON tool definitions tokenize at
297+ roughly 2.4 characters per token — schema punctuation is token-dense.
298+ </p>
299+ </div>
300+ </section>
301+
302+ {/* ─── TAKEAWAYS ─── */}
303+ <section class="border-t border-rule px-6 py-14 sm:px-10 sm:py-20">
304+ <div class="mx-auto max-w-[1100px]">
305+ <span class="eyebrow">If you ship an MCP server</span>
306+ <h2 class="section-title mt-4 max-w-[24ch]">Design for the weakest channel</h2>
307+ <div class="cap-grid mt-10" style="grid-template-columns:repeat(auto-fit,minmax(240px,1fr));">
308+ <div class="cap-card">
309+ <span class="cap-num">01</span>
310+ <h3>The name is the only universal channel</h3>
311+ <p>
312+ Every harness that shows tools at all shows the name. Descriptions reach only the
313+ inliners; schemas effectively reach only the inliners. If information must arrive,
314+ put it in the name.
315+ </p>
316+ </div>
317+ <div class="cap-card">
318+ <span class="cap-num">02</span>
319+ <h3>Schema tricks don't travel</h3>
320+ <p>
321+ An enum of options inside a parameter looks clever and costs less — and is invisible
322+ in Cursor and Codex, the two harnesses where you'd need it most.
323+ </p>
324+ </div>
325+ <div class="cap-card">
326+ <span class="cap-num">03</span>
327+ <h3>Keep definitions small anyway</h3>
328+ <p>
329+ Where tools are inlined, every byte is paid on every turn, multiplied by tool count.
330+ One shared description sentence and a single parameter cut this surface by more than half.
331+ </p>
332+ </div>
333+ <div class="cap-card">
334+ <span class="cap-num">04</span>
335+ <h3>Measure with provider usage, not estimates</h3>
336+ <p>
337+ Client-side token meters and character counts both mislead. Run the same session
338+ twice and diff what the provider bills — that number is the truth.
339+ </p>
340+ </div>
341+ </div>
342+ </div>
343+ </section>
344+
345+ {/* ─── BOTTOM CTA ─── */}
346+ <section class="border-t border-rule py-16 sm:py-20">
347+ <div class="mx-auto max-w-[720px] px-6 text-center sm:px-10">
348+ <h2 class="section-title mx-auto mb-7 max-w-[20ch]">
349+ Connect any agent to <span class="serif-italic">everything</span>.
350+ </h2>
351+ <div class="flex flex-wrap items-center justify-center gap-3">
352+ <a href="/#install" class="btn-primary btn-primary--hero">Get started →</a>
353+ <a href={GH} class="btn-secondary">Star on GitHub</a>
354+ </div>
355+ </div>
356+ </section>
357+
358+ {/* ─── FOOTER ─── */}
359+ <footer class="border-t border-rule py-10">
360+ <div
361+ class="mx-auto flex max-w-[1200px] flex-col items-start justify-between gap-6 px-6 sm:flex-row sm:items-center sm:px-10"
362+ >
363+ <div class="flex items-center gap-2.5">
364+ <img src="/favicon-192.png" alt="" class="h-4 w-4 opacity-80" />
365+ <span class="text-[13px] font-medium text-ink-2">executor</span>
366+ <span class="ml-2 text-[12px] text-ink-3">© {new Date().getFullYear()}</span>
367+ </div>
368+ <div class="flex items-center gap-6 text-[13px] text-ink-2">
369+ <a href="/blog" class="transition-colors hover:text-ink">Blog</a>
370+ <a href="https://executor.sh/docs" class="transition-colors hover:text-ink">Docs</a>
371+ <a href={GH} class="transition-colors hover:text-ink">GitHub</a>
372+ <a href="/privacy" class="transition-colors hover:text-ink">Privacy</a>
373+ <a href="/terms" class="transition-colors hover:text-ink">Terms</a>
374+ </div>
375+ </div>
376+ </footer>
377+ </div>
378+ </main>
379+</Layout>
380+
381+<style>
382+ .mono-inline {
383+ font-family: var(--font-mono);
384+ font-size: 0.88em;
385+ background: var(--color-surface-2);
386+ border: 1px solid var(--color-rule);
387+ border-radius: 4px;
388+ padding: 1px 5px;
389+ white-space: nowrap;
390+ }
391+
392+ /* ── perspective cards ── */
393+ .persp {
394+ padding: 0;
395+ overflow: hidden;
396+ }
397+ .persp-head {
398+ display: flex;
399+ align-items: baseline;
400+ justify-content: space-between;
401+ gap: 16px;
402+ padding: 18px 24px;
403+ border-bottom: 1px solid var(--color-rule);
404+ }
405+ .persp-name {
406+ font-weight: 600;
407+ font-size: 17px;
408+ letter-spacing: -0.01em;
409+ color: var(--color-ink);
410+ }
411+ .persp-family {
412+ display: block;
413+ margin-top: 3px;
414+ font-family: var(--font-mono);
415+ font-size: 11px;
416+ letter-spacing: 0.08em;
417+ text-transform: uppercase;
418+ color: var(--color-ink-3);
419+ }
420+ .persp-cost {
421+ font-family: var(--font-mono);
422+ font-size: 13px;
423+ color: var(--color-ink);
424+ white-space: nowrap;
425+ }
426+ .persp-body {
427+ display: grid;
428+ grid-template-columns: 340px 1fr;
429+ gap: 0;
430+ }
431+ @media (max-width: 820px) {
432+ .persp-body {
433+ grid-template-columns: 1fr;
434+ }
435+ }
436+
437+ /* context-window schematic */
438+ .ctx {
439+ padding: 20px 24px;
440+ border-right: 1px solid var(--color-rule);
441+ display: flex;
442+ flex-direction: column;
443+ gap: 6px;
444+ }
445+ @media (max-width: 820px) {
446+ .ctx {
447+ border-right: 0;
448+ border-bottom: 1px solid var(--color-rule);
449+ }
450+ }
451+ .ctx-label {
452+ font-family: var(--font-mono);
453+ font-size: 10.5px;
454+ letter-spacing: 0.08em;
455+ text-transform: uppercase;
456+ color: var(--color-ink-3);
457+ margin-bottom: 4px;
458+ }
459+ .ctx-block {
460+ border-radius: 6px;
461+ padding: 9px 12px;
462+ font-size: 12.5px;
463+ line-height: 1.45;
464+ }
465+ .ctx-quiet {
466+ border: 1px solid var(--color-rule);
467+ background: var(--color-surface-2);
468+ color: var(--color-ink-2);
469+ }
470+ .ctx-loud {
471+ border: 1px solid var(--color-rule-strong);
472+ background: var(--color-surface);
473+ color: var(--color-ink);
474+ }
475+ .ctx-loud b {
476+ display: block;
477+ font-weight: 600;
478+ font-size: 13px;
479+ }
480+ .ctx-loud span {
481+ display: block;
482+ margin-top: 2px;
483+ color: var(--color-ink-2);
484+ font-size: 12px;
485+ }
486+ .ctx-dashed {
487+ border: 1px dashed var(--color-rule-strong);
488+ color: var(--color-ink-3);
489+ background: transparent;
490+ }
491+ .ctx-never {
492+ border: 1px dashed var(--color-rule);
493+ background: transparent;
494+ color: var(--color-ink-3);
495+ }
496+ .ctx-never b {
497+ display: block;
498+ font-weight: 500;
499+ font-size: 13px;
500+ text-decoration: line-through;
501+ color: var(--color-ink-3);
502+ }
503+ .ctx-never span {
504+ display: block;
505+ margin-top: 2px;
506+ font-size: 12px;
507+ }
508+
509+ .persp-detail {
510+ padding: 20px 24px;
511+ display: flex;
512+ flex-direction: column;
513+ gap: 14px;
514+ min-width: 0;
515+ }
516+ .persp-detail .code-window {
517+ margin: 0;
518+ }
519+ .persp-detail .code-body {
520+ font-size: 12px;
521+ line-height: 1.6;
522+ overflow-x: auto;
523+ }
524+ .persp-path {
525+ font-size: 13.5px;
526+ line-height: 1.6;
527+ color: var(--color-ink-2);
528+ }
529+ .persp-path-label {
530+ display: block;
531+ font-family: var(--font-mono);
532+ font-size: 10.5px;
533+ letter-spacing: 0.08em;
534+ text-transform: uppercase;
535+ color: var(--color-ink-3);
536+ margin-bottom: 3px;
537+ }
538+
539+ /* ── cost chart ── */
540+ .cost-card {
541+ padding: 10px 24px;
542+ }
543+ .cost-row {
544+ display: grid;
545+ grid-template-columns: 240px 1fr 84px;
546+ align-items: center;
547+ gap: 16px;
548+ padding: 11px 0;
549+ }
550+ .cost-row + .cost-row {
551+ border-top: 1px solid var(--color-rule);
552+ }
553+ @media (max-width: 640px) {
554+ .cost-row {
555+ grid-template-columns: 1fr 84px;
556+ }
557+ .cost-track {
558+ display: none;
559+ }
560+ }
561+ .cost-name {
562+ font-size: 14px;
563+ font-weight: 500;
564+ color: var(--color-ink);
565+ }
566+ .cost-detail {
567+ display: block;
568+ font-family: var(--font-mono);
569+ font-size: 11px;
570+ color: var(--color-ink-3);
571+ margin-top: 2px;
572+ }
573+ .cost-track {
574+ position: relative;
575+ height: 20px;
576+ }
577+ .cost-bar {
578+ position: absolute;
579+ top: 3px;
580+ bottom: 3px;
581+ left: 0;
582+ background: var(--color-ink);
583+ border-radius: 0 4px 4px 0;
584+ min-width: 3px;
585+ }
586+ .cost-bar--zero {
587+ background: transparent;
588+ border-left: 2px solid var(--color-ink);
589+ border-radius: 0;
590+ min-width: 0;
591+ }
592+ .cost-val {
593+ font-family: var(--font-mono);
594+ font-size: 13px;
595+ text-align: right;
596+ color: var(--color-ink);
597+ }
598+</style>

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t