History

sync shiki init with typescript, lazy-load other langs; fix sidebar border

Rhys Sullivan committed 7 months ago

Commit 6bc94ea1 parent 0d978a24 files · +94 −89Browse files

Unified · Split

4 files
  1. packages/react/src/components/code-block.tsx +8 −17
  2. packages/react/src/components/expandable-code-block.tsx +14 −32
  3. packages/react/src/components/tool-tree.tsx +2 −2
  4. packages/react/src/lib/shiki.ts +70 −38
packages/react/src/components/code-block.tsx+8−17
@@ -3,6 +3,7 @@ import { jsx, jsxs, Fragment } from "react/jsx-runtime"; · 2 unmodified lines
3import { toJsxRuntime } from "hast-util-to-jsx-runtime";3import { toJsxRuntime } from "hast-util-to-jsx-runtime";
4import {4import {
5 getHighlighter,5 getHighlighter,
6 ensureLang,
6 resolveLang,7 resolveLang,
7 useResolvedShikiTheme,8 useResolvedShikiTheme,
8 type ShikiThemeProp,9 type ShikiThemeProp,
@@ -62,26 +63,16 @@ const CheckIcon = () => ( · 53 unmodified lines
62// ---------------------------------------------------------------------------63// ---------------------------------------------------------------------------
63 64
64function useHighlighted(code: string, lang: string, theme: SupportedTheme): ReactNode | null {65function useHighlighted(code: string, lang: string, theme: SupportedTheme): ReactNode | null {
65 const [highlighted, setHighlighted] = useState<ReactNode | null>(null);66 const [, setTick] = useState(0);
67 const resolvedLang = (resolveLang(lang) ?? "json") as Parameters<typeof ensureLang>[0];
66 68
67 useEffect(() => {69 const isReady = ensureLang(resolvedLang, () => setTick((t) => t + 1));
68 let cancelled = false;
69 70
70 getHighlighter().then((highlighter) => {71 if (!isReady) return null;
71 if (cancelled) return;
72 72
73 const hast = highlighter.codeToHast(code, { lang, theme });73 const highlighter = getHighlighter();
74 const nodes = toJsxRuntime(hast, { jsx, jsxs, Fragment });74 const hast = highlighter.codeToHast(code, { lang: resolvedLang, theme });
75 75 return toJsxRuntime(hast, { jsx, jsxs, Fragment });
76 if (!cancelled) setHighlighted(nodes);
77 });
78
79 return () => {
80 cancelled = true;
81 };
82 }, [code, lang, theme]);
83
84 return highlighted;
85}76}
86 77
87// ---------------------------------------------------------------------------78// ---------------------------------------------------------------------------
packages/react/src/components/expandable-code-block.tsx+14−32
@@ -1,4 +1,4 @@
1import { useCallback, useEffect, useMemo, useState, startTransition } from "react";1import { useCallback, useMemo, useState } from "react";
2import {2import {
3 getHighlighter,3 getHighlighter,
4 useResolvedShikiTheme,4 useResolvedShikiTheme,
@@ -103,27 +103,13 @@ const CheckIcon = () => ( · 98 unmodified lines
103// Shiki tokenization hook — non-blocking103// Shiki tokenization hook — non-blocking
104// ---------------------------------------------------------------------------104// ---------------------------------------------------------------------------
105 105
106function useTokens(code: string, theme: SupportedTheme): ThemedToken[][] | null {106function useTokens(code: string, theme: SupportedTheme): ThemedToken[][] {
107 const [tokens, setTokens] = useState<ThemedToken[][] | null>(null);107 const highlighter = getHighlighter();
108 108 const result = highlighter.codeToTokens(code, {
109 useEffect(() => {109 lang: "typescript",
110 let cancelled = false;110 theme,
111 getHighlighter().then((highlighter) => {111 });
112 if (cancelled) return;112 return result.tokens;
113 const result = highlighter.codeToTokens(code, {
114 lang: "typescript",
115 theme,
116 });
117 if (!cancelled) {
118 startTransition(() => setTokens(result.tokens));
119 }
120 });
121 return () => {
122 cancelled = true;
123 };
124 }, [code, theme]);
125
126 return tokens;
127}113}
128 114
129// ---------------------------------------------------------------------------115// ---------------------------------------------------------------------------
@@ -391,16 +377,12 @@ export function ExpandableCodeBlock(props: { · 261 unmodified lines
391 377
392 <pre className="overflow-auto p-3 font-mono text-[0.75rem] leading-6 !bg-transparent">378 <pre className="overflow-auto p-3 font-mono text-[0.75rem] leading-6 !bg-transparent">
393 <code>379 <code>
394 {tokens ? (380 <HighlightedCode
395 <HighlightedCode381 tokens={tokens}
396 tokens={tokens}382 clickableNames={clickableNames}
397 clickableNames={clickableNames}383 onToggle={handleToggle}
398 onToggle={handleToggle}384 expanded={allExpanded}
399 expanded={allExpanded}385 />
400 />
401 ) : (
402 <span className="text-foreground/60">{displayCode}</span>
403 )}
404 </code>386 </code>
405 </pre>387 </pre>
406 </div>388 </div>
packages/react/src/components/tool-tree.tsx+2−2
@@ -253,8 +253,8 @@ export function ToolTree(props: { · 252 unmodified lines
253 }, [selectedToolId, rows]);253 }, [selectedToolId, rows]);
254 254
255 return (255 return (
256 <div className="flex h-full min-h-0 flex-col p-3">256 <div className="flex h-full min-h-0 flex-col">
257 <CardStack className="flex min-h-0 flex-1 flex-col">257 <CardStack className="flex min-h-0 flex-1 flex-col rounded-none border-0">
258 {/* Search header */}258 {/* Search header */}
259 <div className="flex shrink-0 items-center gap-2 border-b border-border/60 px-3 py-2">259 <div className="flex shrink-0 items-center gap-2 border-b border-border/60 px-3 py-2">
260 <SearchIcon aria-hidden className="size-3.5 shrink-0 text-muted-foreground" />260 <SearchIcon aria-hidden className="size-3.5 shrink-0 text-muted-foreground" />
packages/react/src/lib/shiki.ts+70−38
@@ -1,9 +1,21 @@
1import { createHighlighterCore, type HighlighterCore, type LanguageInput } from "shiki/core";1import { createHighlighterCoreSync, type HighlighterCore } from "shiki/core";
2import { createJavaScriptRegexEngine } from "shiki/engine/javascript";2import { createJavaScriptRegexEngine } from "shiki/engine/javascript";
3import { useIsDark } from "../hooks/use-is-dark";3import { useIsDark } from "../hooks/use-is-dark";
4 4
5// ---------------------------------------------------------------------------5// ---------------------------------------------------------------------------
6// Supported languages — explicit imports to avoid bundling all grammars6// Eagerly loaded languages (sync — available immediately)
7// ---------------------------------------------------------------------------
8
9import langTypescript from "@shikijs/langs/typescript";
10import langJavascript from "@shikijs/langs/javascript";
11import langTsx from "@shikijs/langs/tsx";
12import langJsx from "@shikijs/langs/jsx";
13import langJson from "@shikijs/langs/json";
14import githubDark from "@shikijs/themes/github-dark";
15import githubLight from "@shikijs/themes/github-light";
16
17// ---------------------------------------------------------------------------
18// Lazily loaded languages — imported on first use
7// ---------------------------------------------------------------------------19// ---------------------------------------------------------------------------
8 20
9const SUPPORTED_LANGS = [21const SUPPORTED_LANGS = [
@@ -61,13 +73,10 @@ const LANG_ALIASES: Record<string, SupportedLang> = { · 51 unmodified lines
61 yml: "yaml",73 yml: "yaml",
62};74};
63 75
64const LANG_LOADERS: Record<SupportedLang, () => LanguageInput> = {76const LAZY_LANG_LOADERS: Partial<Record<SupportedLang, () => Promise<unknown>>> = {
65 json: () => import("@shikijs/langs/json"),
66 xml: () => import("@shikijs/langs/xml"),77 xml: () => import("@shikijs/langs/xml"),
67 yaml: () => import("@shikijs/langs/yaml"),78 yaml: () => import("@shikijs/langs/yaml"),
68 shellscript: () => import("@shikijs/langs/shellscript"),79 shellscript: () => import("@shikijs/langs/shellscript"),
69 typescript: () => import("@shikijs/langs/typescript"),
70 javascript: () => import("@shikijs/langs/javascript"),
71 python: () => import("@shikijs/langs/python"),80 python: () => import("@shikijs/langs/python"),
72 html: () => import("@shikijs/langs/html"),81 html: () => import("@shikijs/langs/html"),
73 css: () => import("@shikijs/langs/css"),82 css: () => import("@shikijs/langs/css"),
@@ -84,8 +93,6 @@ const LANG_LOADERS: Record<SupportedLang, () => LanguageInput> = { · 10 unmodified lines
84 c: () => import("@shikijs/langs/c"),93 c: () => import("@shikijs/langs/c"),
85 cpp: () => import("@shikijs/langs/cpp"),94 cpp: () => import("@shikijs/langs/cpp"),
86 csharp: () => import("@shikijs/langs/csharp"),95 csharp: () => import("@shikijs/langs/csharp"),
87 tsx: () => import("@shikijs/langs/tsx"),
88 jsx: () => import("@shikijs/langs/jsx"),
89 toml: () => import("@shikijs/langs/toml"),96 toml: () => import("@shikijs/langs/toml"),
90 dockerfile: () => import("@shikijs/langs/dockerfile"),97 dockerfile: () => import("@shikijs/langs/dockerfile"),
91 diff: () => import("@shikijs/langs/diff"),98 diff: () => import("@shikijs/langs/diff"),
@@ -121,8 +128,10 @@ export function useResolvedShikiTheme(theme?: ShikiThemeProp): SupportedTheme { · 29 unmodified lines
121 128
122export function resolveLang(lang: string): SupportedLang | null {129export function resolveLang(lang: string): SupportedLang | null {
123 const l = lang.trim().toLowerCase();130 const l = lang.trim().toLowerCase();
124 if (l in LANG_LOADERS) return l as SupportedLang;131 if (supportedSet.has(l)) {
125 if (l in LANG_ALIASES) return LANG_ALIASES[l]!;132 if (l in LANG_ALIASES) return LANG_ALIASES[l]!;
133 return l as SupportedLang;
134 }
126 return null;135 return null;
127}136}
128 137
@@ -131,22 +140,52 @@ export function isSupportedLang(lang: string): boolean { · 2 unmodified lines
131}140}
132 141
133// ---------------------------------------------------------------------------142// ---------------------------------------------------------------------------
134// Shared singleton highlighter — lazy, created on first use143// Shared singleton highlighter — synchronous, created eagerly with core langs
135// ---------------------------------------------------------------------------144// ---------------------------------------------------------------------------
136 145
137const jsEngine = createJavaScriptRegexEngine({ forgiving: true });146const highlighter: HighlighterCore = createHighlighterCoreSync({
147 themes: [githubDark, githubLight],
148 langs: [langTypescript, langJavascript, langTsx, langJsx, langJson],
149 engine: createJavaScriptRegexEngine({ forgiving: true }),
150});
138 151
139let _promise: Promise<HighlighterCore> | null = null;152export function getHighlighter(): HighlighterCore {
153 return highlighter;
154}
140 155
141export function getHighlighter(): Promise<HighlighterCore> {156/**
142 if (!_promise) {157 * Ensure a language is loaded into the highlighter. Returns true if the
143 _promise = createHighlighterCore({158 * language is ready for synchronous use, false if it needs to be loaded
144 themes: [import("@shikijs/themes/github-dark"), import("@shikijs/themes/github-light")],159 * asynchronously (in which case `onLoaded` will be called when ready).
145 langs: Object.values(LANG_LOADERS).map((loader) => loader()),160 */
146 engine: jsEngine,161const loadingLangs = new Set<string>();
162
163export function ensureLang(lang: SupportedLang, onLoaded?: () => void): boolean {
164 const loaded = highlighter.getLoadedLanguages();
165 if (loaded.includes(lang)) return true;
166
167 const loader = LAZY_LANG_LOADERS[lang];
168 if (!loader) return true; // Not a lazy lang, must be already loaded
169
170 if (!loadingLangs.has(lang)) {
171 loadingLangs.add(lang);
172 void loader().then((mod: any) => {
173 const registration = mod.default ?? mod;
174 highlighter.loadLanguageSync(registration);
175 loadingLangs.delete(lang);
176 onLoaded?.();
177 });
178 } else if (onLoaded) {
179 const l = loader;
180 void l().then((mod: any) => {
181 const registration = mod.default ?? mod;
182 if (!highlighter.getLoadedLanguages().includes(lang)) {
183 highlighter.loadLanguageSync(registration);
184 }
185 onLoaded();
147 });186 });
148 }187 }
149 return _promise;188 return false;
150}189}
151 190
152// ---------------------------------------------------------------------------191// ---------------------------------------------------------------------------
@@ -156,7 +195,6 @@ export function getHighlighter(): Promise<HighlighterCore> { · 3 unmodified lines
156import type { CodeHighlighterPlugin, ThemeInput } from "streamdown";195import type { CodeHighlighterPlugin, ThemeInput } from "streamdown";
157 196
158const tokensCache = new Map<string, unknown>();197const tokensCache = new Map<string, unknown>();
159const pendingCallbacks = new Map<string, Set<(result: unknown) => void>>();
160 198
161/**199/**
162 * Read the current system color-scheme preference synchronously. Used in200 * Read the current system color-scheme preference synchronously. Used in
@@ -184,30 +222,24 @@ export function createCodeHighlighterPlugin(): CodeHighlighterPlugin { · 21 unmodified lines
184 const cached = tokensCache.get(key);222 const cached = tokensCache.get(key);
185 if (cached) return cached as never;223 if (cached) return cached as never;
186 224
187 if (callback) {225 const isReady = ensureLang(lang, () => {
188 if (!pendingCallbacks.has(key)) {226 // Language just loaded — highlight and notify via callback
189 pendingCallbacks.set(key, new Set());
190 }
191 pendingCallbacks.get(key)!.add(callback as (result: unknown) => void);
192 }
193
194 void getHighlighter().then((highlighter) => {
195 if (tokensCache.has(key)) {
196 const result = tokensCache.get(key);
197 pendingCallbacks.get(key)?.forEach((cb) => cb(result));
198 pendingCallbacks.delete(key);
199 return;
200 }
201 const result = highlighter.codeToTokens(options.code, {227 const result = highlighter.codeToTokens(options.code, {
202 lang,228 lang,
203 themes: { light: activeTheme, dark: activeTheme },229 themes: { light: activeTheme, dark: activeTheme },
204 });230 });
205 tokensCache.set(key, result);231 tokensCache.set(key, result);
206 pendingCallbacks.get(key)?.forEach((cb) => cb(result));232 callback?.(result as never);
207 pendingCallbacks.delete(key);
208 });233 });
209 234
210 return null;235 if (!isReady) return null;
236
237 const result = highlighter.codeToTokens(options.code, {
238 lang,
239 themes: { light: activeTheme, dark: activeTheme },
240 });
241 tokensCache.set(key, result);
242 return result as never;
211 },243 },
212 };244 };
213}245}