import { directiveFromMarkdown } from "mdast-util-directive"; import { directive } from "micromark-extension-directive"; import { markdownLineEnding, unicodePunctuation, unicodeWhitespace, } from "micromark-util-character"; import type { Construct, Extension, Tokenizer } from "micromark-util-types"; import remarkParse from "remark-parse"; import { unified, type Processor } from "unified"; import { codexArtifactTemplatePresentationLabel, resolveCodexArtifactTemplate, type CodexArtifactTemplate, } from "./codexArtifactTemplates.ts"; import { codexFileCitationMarkdown, resolveCodexFileCitationLink } from "./codexFileCitations.ts"; const COLON = 58; const DASH = 45; const UNDERSCORE = 95; const CODEX_FILE_CITATION_NAME = "codex-file-citation"; const CODEX_ARTIFACT_TEMPLATE_NAME = "artifact-template"; export const CODEX_ARTIFACT_TEMPLATE_HAST_PROPERTIES = [ "dataCodexArtifactTemplate", "dataArtifactKind", "dataDisplayName", "dataGalleryKind", "dataSkillDirectory", "dataSkillName", ] as const; interface MarkdownPosition { readonly start: { readonly offset?: number }; readonly end: { readonly offset?: number }; } interface MarkdownAstNode { type?: string; name?: string; value?: string; url?: string; attributes?: Readonly>; position?: MarkdownPosition; data?: { codexArtifactTemplate?: CodexArtifactTemplate; codexFileCitationMarkdown?: string; hName?: string; hProperties?: Record; }; children?: MarkdownAstNode[]; } interface MarkdownFile { readonly value: unknown; } export type CodexArtifactTemplateMarkdownSegment = | { readonly kind: "markdown"; readonly markdown: string; readonly sourceOffset: number } | { readonly kind: "artifact-template"; readonly sourceOffset: number; readonly template: CodexArtifactTemplate; }; function asConstruct(value: Construct | Construct[] | undefined, label: string): Construct { const construct = Array.isArray(value) ? value[0] : value; if (!construct) throw new Error(`Missing ${label} directive construct`); return construct; } function directiveNameEnds(code: number | null): boolean { return ( code === null || markdownLineEnding(code) || unicodeWhitespace(code) || (unicodePunctuation(code) && code !== DASH && code !== UNDERSCORE) ); } function directiveNameGate(markerCount: number, name: string): Construct { const tokenize: Tokenizer = (effects, ok, nok) => { let markerIndex = 0; let nameIndex = 0; return marker; function marker(code: number | null) { if (code !== COLON) return nok(code); if (markerIndex === 0) effects.enter("data"); effects.consume(code); markerIndex += 1; return markerIndex === markerCount ? nameCharacter : marker; } function nameCharacter(code: number | null) { if (code !== name.charCodeAt(nameIndex)) return nok(code); effects.consume(code); nameIndex += 1; return nameIndex === name.length ? afterName : nameCharacter; } function afterName(code: number | null) { effects.exit("data"); return directiveNameEnds(code) ? ok(code) : nok(code); } }; return { partial: true, tokenize }; } function restrictedDirective(construct: Construct, markerCount: number, name: string): Construct { const gate = directiveNameGate(markerCount, name); return { ...construct, tokenize(effects, ok, nok) { return effects.check(gate, construct.tokenize.call(this, effects, ok, nok), nok); }, }; } function codexDirectiveSyntax(): Extension { const genericSyntax = directive(); const textDirective = asConstruct(genericSyntax.text?.[COLON], CODEX_FILE_CITATION_NAME); const flowDirectives = genericSyntax.flow?.[COLON]; const leafDirective = Array.isArray(flowDirectives) ? flowDirectives.find((construct) => construct.concrete !== true) : flowDirectives; if (!leafDirective) throw new Error(`Missing ${CODEX_ARTIFACT_TEMPLATE_NAME} directive construct`); return { text: { [COLON]: restrictedDirective(textDirective, 1, CODEX_FILE_CITATION_NAME), }, flow: { [COLON]: restrictedDirective(leafDirective, 2, CODEX_ARTIFACT_TEMPLATE_NAME), }, }; } const CODEX_DIRECTIVE_SYNTAX = codexDirectiveSyntax(); const CODEX_DIRECTIVE_FROM_MARKDOWN = directiveFromMarkdown(); function sourceForNode(node: MarkdownAstNode, source: string): string { const start = node.position?.start.offset; const end = node.position?.end.offset; return start === undefined || end === undefined ? "" : source.slice(start, end); } function sourceForDirective(node: MarkdownAstNode, source: string, marker: ":" | "::"): string { const prefix = `${marker}${node.name ?? ""}`; const slicedSource = sourceForNode(node, source); if (slicedSource.startsWith(prefix)) return slicedSource; const attributes = Object.entries(node.attributes ?? {}).map(([name, value]) => value === null ? name : `${name}=${JSON.stringify(value)}`, ); return `${prefix}${attributes.length === 0 ? "" : `{${attributes.join(" ")}}`}`; } function restoreTextDirective(node: MarkdownAstNode, source: string): void { node.type = "text"; node.value = sourceForDirective(node, source, ":"); delete node.name; delete node.attributes; delete node.url; delete node.data; delete node.children; } function restoreLeafDirective(node: MarkdownAstNode, source: string): void { const value = sourceForDirective(node, source, "::"); node.type = "paragraph"; node.children = [ { type: "text", value, ...(node.position === undefined ? {} : { position: node.position }) }, ]; delete node.name; delete node.attributes; delete node.value; delete node.url; delete node.data; } function renderFileCitation(node: MarkdownAstNode, source: string, insideLink: boolean): void { const citation = resolveCodexFileCitationLink(node.attributes); if (!citation || insideLink) { restoreTextDirective(node, source); return; } node.type = "link"; node.url = citation.href; node.children = [{ type: "text", value: citation.label }]; node.data = { codexFileCitationMarkdown: codexFileCitationMarkdown(citation) }; delete node.name; delete node.attributes; delete node.value; } function renderArtifactTemplate(node: MarkdownAstNode, source: string): void { const template = resolveCodexArtifactTemplate(node.attributes); if (!template) { restoreLeafDirective(node, source); return; } node.type = "paragraph"; node.children = []; node.data = { codexArtifactTemplate: template, hName: "div", hProperties: { dataCodexArtifactTemplate: "true", dataArtifactKind: template.artifactKind, dataDisplayName: template.displayName, ...(template.galleryKind === undefined ? {} : { dataGalleryKind: template.galleryKind }), dataSkillDirectory: template.skillDirectory, dataSkillName: template.skillName, }, }; delete node.name; delete node.attributes; delete node.value; delete node.url; } function transformCodexDirectives(node: MarkdownAstNode, source: string, insideLink = false): void { if (node.type === "textDirective" && node.name === CODEX_FILE_CITATION_NAME) { renderFileCitation(node, source, insideLink); return; } if (node.type === "leafDirective" && node.name === CODEX_ARTIFACT_TEMPLATE_NAME) { renderArtifactTemplate(node, source); return; } const childrenInsideLink = insideLink || node.type === "link" || node.type === "linkReference"; for (const child of node.children ?? []) { transformCodexDirectives(child, source, childrenInsideLink); } } /** Adds grammar only for the two directives emitted by Codex, then renders them as mdast. */ function attachCodexDirectives(this: Processor) { const data = this.data(); const micromarkExtensions = data.micromarkExtensions ?? (data.micromarkExtensions = []); const fromMarkdownExtensions = data.fromMarkdownExtensions ?? (data.fromMarkdownExtensions = []); micromarkExtensions.push(CODEX_DIRECTIVE_SYNTAX); fromMarkdownExtensions.push(CODEX_DIRECTIVE_FROM_MARKDOWN); return (tree: unknown, file: MarkdownFile) => { transformCodexDirectives(tree as MarkdownAstNode, String(file.value)); }; } export const remarkCodexDirectives = attachCodexDirectives; const directiveParser = unified().use(remarkParse).use(remarkCodexDirectives).freeze(); function parseCodexMarkdown(markdown: string): MarkdownAstNode { return directiveParser.runSync(directiveParser.parse(markdown), { value: markdown, }) as MarkdownAstNode; } interface DirectiveMatch { readonly start: number; readonly end: number; readonly markdown?: string; readonly template?: CodexArtifactTemplate; } function collectDirectiveMatches(node: MarkdownAstNode, matches: DirectiveMatch[]): void { const start = node.position?.start.offset; const end = node.position?.end.offset; if (start !== undefined && end !== undefined) { if (node.data?.codexFileCitationMarkdown !== undefined) { matches.push({ start, end, markdown: node.data.codexFileCitationMarkdown }); return; } if (node.data?.codexArtifactTemplate !== undefined) { matches.push({ start, end, template: node.data.codexArtifactTemplate }); return; } } for (const child of node.children ?? []) collectDirectiveMatches(child, matches); } function renderDirectiveMatches( markdown: string, replacementFor: (match: DirectiveMatch) => string | undefined, ): string { const matches: DirectiveMatch[] = []; collectDirectiveMatches(parseCodexMarkdown(markdown), matches); let rendered = markdown; for (const match of matches.sort((left, right) => right.start - left.start)) { const replacement = replacementFor(match); if (replacement !== undefined) { rendered = rendered.slice(0, match.start) + replacement + rendered.slice(match.end); } } return rendered; } /** Native Markdown renderers use this adapter because they cannot consume a Remark tree. */ export function renderCodexFileCitationsAsMarkdown(markdown: string): string { if (!markdown.includes(`:${CODEX_FILE_CITATION_NAME}`)) return markdown; return renderDirectiveMatches(markdown, (match) => match.markdown); } /** Matches the Markdown emitted when users copy rendered Codex directive UI. */ export function renderCodexDirectivesForCopy(markdown: string): string { if ( !markdown.includes(`:${CODEX_FILE_CITATION_NAME}`) && !markdown.includes(`::${CODEX_ARTIFACT_TEMPLATE_NAME}`) ) { return markdown; } return renderDirectiveMatches(markdown, (match) => { if (match.markdown !== undefined) return match.markdown; if (match.template === undefined) return undefined; return `${match.template.displayName} (${codexArtifactTemplatePresentationLabel(match.template.artifactKind)})`; }); } /** Native renderers split cards out because they cannot host a view inside Markdown text. */ export function splitCodexArtifactTemplateMarkdown( markdown: string, ): ReadonlyArray { if (!markdown.includes(`::${CODEX_ARTIFACT_TEMPLATE_NAME}`)) { return [{ kind: "markdown", markdown, sourceOffset: 0 }]; } const matches: DirectiveMatch[] = []; collectDirectiveMatches(parseCodexMarkdown(markdown), matches); const templates = matches .filter( (match): match is DirectiveMatch & { readonly template: CodexArtifactTemplate } => match.template !== undefined, ) .sort((left, right) => left.start - right.start); if (templates.length === 0) { return [{ kind: "markdown", markdown, sourceOffset: 0 }]; } const segments: CodexArtifactTemplateMarkdownSegment[] = []; let cursor = 0; for (const match of templates) { if (match.start > cursor) { segments.push({ kind: "markdown", markdown: markdown.slice(cursor, match.start), sourceOffset: cursor, }); } segments.push({ kind: "artifact-template", sourceOffset: match.start, template: match.template, }); cursor = match.end; } if (cursor < markdown.length) { segments.push({ kind: "markdown", markdown: markdown.slice(cursor), sourceOffset: cursor }); } return segments; } export function artifactTemplateFromHastProperties( properties: Readonly> | null | undefined, ): CodexArtifactTemplate | null { if (properties?.dataCodexArtifactTemplate !== "true") return null; const stringProperty = (name: string) => { const value = properties[name]; return typeof value === "string" ? value : undefined; }; return resolveCodexArtifactTemplate({ artifact_kind: stringProperty("dataArtifactKind"), display_name: stringProperty("dataDisplayName"), gallery_kind: stringProperty("dataGalleryKind"), skill_directory: stringProperty("dataSkillDirectory"), skill_name: stringProperty("dataSkillName"), }); }