use crate::{ markdown_elements::*, markdown_minifier::{Minifier, MinifierOptions},};use async_recursion::async_recursion;use collections::FxHashMap;use gpui::{DefiniteLength, FontWeight, px, relative};use html5ever::{ParseOpts, local_name, parse_document, tendril::TendrilSink};use language::LanguageRegistry;use markup5ever_rcdom::RcDom;use pulldown_cmark::{Alignment, Event, Options, Parser, Tag, TagEnd};use std::{ cell::RefCell, collections::HashMap, mem, ops::Range, path::PathBuf, rc::Rc, sync::Arc, vec,};use ui::SharedString; pub async fn parse_markdown( markdown_input: &str, file_location_directory: Option<PathBuf>, language_registry: Option<Arc<LanguageRegistry>>,) -> ParsedMarkdown { let mut options = Options::all(); options.remove(pulldown_cmark::Options::ENABLE_DEFINITION_LIST); let parser = Parser::new_ext(markdown_input, options); let parser = MarkdownParser::new( parser.into_offset_iter().collect(), file_location_directory, language_registry, ); let renderer = parser.parse_document().await; ParsedMarkdown { children: renderer.parsed, }} fn cleanup_html(source: &str) -> Vec<u8> { let mut writer = std::io::Cursor::new(Vec::new()); let mut reader = std::io::Cursor::new(source); let mut minify = Minifier::new( &mut writer, MinifierOptions { omit_doctype: true, collapse_whitespace: true, ..Default::default() }, ); if let Ok(()) = minify.minify(&mut reader) { writer.into_inner() } else { source.bytes().collect() }} struct MarkdownParser<'a> { tokens: Vec<(Event<'a>, Range<usize>)>, /// The current index in the tokens array cursor: usize, /// The blocks that we have successfully parsed so far parsed: Vec<ParsedMarkdownElement>, file_location_directory: Option<PathBuf>, language_registry: Option<Arc<LanguageRegistry>>,} #[derive(Debug)]struct ParseHtmlNodeContext { list_item_depth: u16,} impl Default for ParseHtmlNodeContext { fn default() -> Self { Self { list_item_depth: 1 } }} struct MarkdownListItem { content: Vec<ParsedMarkdownElement>, item_type: ParsedMarkdownListItemType,} impl Default for MarkdownListItem { fn default() -> Self { Self { content: Vec::new(), item_type: ParsedMarkdownListItemType::Unordered, } }} impl<'a> MarkdownParser<'a> { fn new( tokens: Vec<(Event<'a>, Range<usize>)>, file_location_directory: Option<PathBuf>, language_registry: Option<Arc<LanguageRegistry>>, ) -> Self { Self { tokens, file_location_directory, language_registry, cursor: 0, parsed: vec![], } } fn eof(&self) -> bool { if self.tokens.is_empty() { return true; } self.cursor >= self.tokens.len() - 1 } fn peek(&self, steps: usize) -> Option<&(Event<'_>, Range<usize>)> { if self.eof() || (steps + self.cursor) >= self.tokens.len() { return self.tokens.last(); } self.tokens.get(self.cursor + steps) } fn previous(&self) -> Option<&(Event<'_>, Range<usize>)> { if self.cursor == 0 || self.cursor > self.tokens.len() { return None; } self.tokens.get(self.cursor - 1) } fn current(&self) -> Option<&(Event<'_>, Range<usize>)> { self.peek(0) } fn current_event(&self) -> Option<&Event<'_>> { self.current().map(|(event, _)| event) } fn is_text_like(event: &Event) -> bool { match event { Event::Text(_) // Represent an inline code block | Event::Code(_) | Event::Html(_) | Event::InlineHtml(_) | Event::FootnoteReference(_) | Event::Start(Tag::Link { .. }) | Event::Start(Tag::Emphasis) | Event::Start(Tag::Strong) | Event::Start(Tag::Strikethrough) | Event::Start(Tag::Image { .. }) => { true } _ => false, } } async fn parse_document(mut self) -> Self { while !self.eof() { if let Some(block) = self.parse_block().await { self.parsed.extend(block); } else { self.cursor += 1; } } self } #[async_recursion] async fn parse_block(&mut self) -> Option<Vec<ParsedMarkdownElement>> { let (current, source_range) = self.current().unwrap(); let source_range = source_range.clone(); match current { Event::Start(tag) => match tag { Tag::Paragraph => { self.cursor += 1; let text = self.parse_text(false, Some(source_range)); Some(vec![ParsedMarkdownElement::Paragraph(text)]) } Tag::Heading { level, .. } => { let level = *level; self.cursor += 1; let heading = self.parse_heading(level); Some(vec![ParsedMarkdownElement::Heading(heading)]) } Tag::Table(alignment) => { let alignment = alignment.clone(); self.cursor += 1; let table = self.parse_table(alignment); Some(vec![ParsedMarkdownElement::Table(table)]) } Tag::List(order) => { let order = *order; self.cursor += 1; let list = self.parse_list(order).await; Some(list) } Tag::BlockQuote(_kind) => { self.cursor += 1; let block_quote = self.parse_block_quote().await; Some(vec![ParsedMarkdownElement::BlockQuote(block_quote)]) } Tag::CodeBlock(kind) => { let language = match kind { pulldown_cmark::CodeBlockKind::Indented => None, pulldown_cmark::CodeBlockKind::Fenced(language) => { if language.is_empty() { None } else { Some(language.to_string()) } } }; self.cursor += 1; let code_block = self.parse_code_block(language).await?; Some(vec![ParsedMarkdownElement::CodeBlock(code_block)]) } Tag::HtmlBlock => { self.cursor += 1; Some(self.parse_html_block().await) } _ => None, }, Event::Rule => { self.cursor += 1; Some(vec![ParsedMarkdownElement::HorizontalRule(source_range)]) } _ => None, } } fn parse_text( &mut self, should_complete_on_soft_break: bool, source_range: Option<Range<usize>>, ) -> MarkdownParagraph { let source_range = source_range.unwrap_or_else(|| { self.current() .map(|(_, range)| range.clone()) .unwrap_or_default() }); let mut markdown_text_like = Vec::new(); let mut text = String::new(); let mut bold_depth = 0; let mut italic_depth = 0; let mut strikethrough_depth = 0; let mut link: Option<Link> = None; let mut image: Option<Image> = None; let mut regions: Vec<(Range<usize>, ParsedRegion)> = vec![]; let mut highlights: Vec<(Range<usize>, MarkdownHighlight)> = vec![]; let mut link_urls: Vec<String> = vec![]; let mut link_ranges: Vec<Range<usize>> = vec![]; loop { if self.eof() { break; } let (current, _) = self.current().unwrap(); let prev_len = text.len(); match current { Event::SoftBreak => { if should_complete_on_soft_break { break; } text.push(' '); } Event::HardBreak => { text.push('\n'); } // We want to ignore any inline HTML tags in the text but keep // the text between them Event::InlineHtml(_) => {} Event::Text(t) => { text.push_str(t.as_ref()); let mut style = MarkdownHighlightStyle::default(); if bold_depth > 0 { style.weight = FontWeight::BOLD; } if italic_depth > 0 { style.italic = true; } if strikethrough_depth > 0 { style.strikethrough = true; } let last_run_len = if let Some(link) = link.clone() { regions.push(( prev_len..text.len(), ParsedRegion { code: false, link: Some(link), }, )); style.link = true; prev_len } else { // Manually scan for links let mut finder = linkify::LinkFinder::new(); finder.kinds(&[linkify::LinkKind::Url]); let mut last_link_len = prev_len; for link in finder.links(t) { let start = prev_len + link.start(); let end = prev_len + link.end(); let range = start..end; link_ranges.push(range.clone()); link_urls.push(link.as_str().to_string()); // If there is a style before we match a link, we have to add this to the highlighted ranges if style != MarkdownHighlightStyle::default() && last_link_len < start { highlights.push(( last_link_len..start, MarkdownHighlight::Style(style.clone()), )); } highlights.push(( range.clone(), MarkdownHighlight::Style(MarkdownHighlightStyle { underline: true, ..style }), )); regions.push(( range.clone(), ParsedRegion { code: false, link: Some(Link::Web { url: link.as_str().to_string(), }), }, )); last_link_len = end; } last_link_len }; if style != MarkdownHighlightStyle::default() && last_run_len < text.len() { let mut new_highlight = true; if let Some((last_range, last_style)) = highlights.last_mut() && last_range.end == last_run_len && last_style == &MarkdownHighlight::Style(style.clone()) { last_range.end = text.len(); new_highlight = false; } if new_highlight { highlights.push(( last_run_len..text.len(), MarkdownHighlight::Style(style.clone()), )); } } } Event::Code(t) => { text.push_str(t.as_ref()); let range = prev_len..text.len(); if link.is_some() { highlights.push(( range.clone(), MarkdownHighlight::Style(MarkdownHighlightStyle { link: true, ..Default::default() }), )); } regions.push(( range, ParsedRegion { code: true, link: link.clone(), }, )); } Event::Start(tag) => match tag { Tag::Emphasis => italic_depth += 1, Tag::Strong => bold_depth += 1, Tag::Strikethrough => strikethrough_depth += 1, Tag::Link { dest_url, .. } => { link = Link::identify( self.file_location_directory.clone(), dest_url.to_string(), ); } Tag::Image { dest_url, .. } => { if !text.is_empty() { let parsed_regions = MarkdownParagraphChunk::Text(ParsedMarkdownText { source_range: source_range.clone(), contents: mem::take(&mut text).into(), highlights: mem::take(&mut highlights), regions: mem::take(&mut regions), }); markdown_text_like.push(parsed_regions); } image = Image::identify( dest_url.to_string(), source_range.clone(), self.file_location_directory.clone(), ); } _ => { break; } }, Event::End(tag) => match tag { TagEnd::Emphasis => italic_depth -= 1, TagEnd::Strong => bold_depth -= 1, TagEnd::Strikethrough => strikethrough_depth -= 1, TagEnd::Link => { link = None; } TagEnd::Image => { if let Some(mut image) = image.take() { if !text.is_empty() { image.set_alt_text(std::mem::take(&mut text).into()); mem::take(&mut highlights); mem::take(&mut regions); } markdown_text_like.push(MarkdownParagraphChunk::Image(image)); } } TagEnd::Paragraph => { self.cursor += 1; break; } _ => { break; } }, _ => { break; } } self.cursor += 1; } if !text.is_empty() { markdown_text_like.push(MarkdownParagraphChunk::Text(ParsedMarkdownText { source_range, contents: text.into(), highlights, regions, })); } markdown_text_like } fn parse_heading(&mut self, level: pulldown_cmark::HeadingLevel) -> ParsedMarkdownHeading { let (_event, source_range) = self.previous().unwrap(); let source_range = source_range.clone(); let text = self.parse_text(true, None); // Advance past the heading end tag self.cursor += 1; ParsedMarkdownHeading { source_range, level: match level { pulldown_cmark::HeadingLevel::H1 => HeadingLevel::H1, pulldown_cmark::HeadingLevel::H2 => HeadingLevel::H2, pulldown_cmark::HeadingLevel::H3 => HeadingLevel::H3, pulldown_cmark::HeadingLevel::H4 => HeadingLevel::H4, pulldown_cmark::HeadingLevel::H5 => HeadingLevel::H5, pulldown_cmark::HeadingLevel::H6 => HeadingLevel::H6, }, contents: text, } } fn parse_table(&mut self, alignment: Vec<Alignment>) -> ParsedMarkdownTable { let (_event, source_range) = self.previous().unwrap(); let source_range = source_range.clone(); let mut header = vec![]; let mut body = vec![]; let mut row_columns = vec![]; let mut in_header = true; let column_alignments = alignment .iter() .map(Self::convert_alignment) .collect::<Vec<_>>(); loop { if self.eof() { break; } let (current, source_range) = self.current().unwrap(); let source_range = source_range.clone(); match current { Event::Start(Tag::TableHead) | Event::Start(Tag::TableRow) | Event::End(TagEnd::TableCell) => {Lines 1–500 of 3235. Load more
use crate::{ markdown_elements::*, markdown_minifier::{Minifier, MinifierOptions}, }; use async_recursion::async_recursion; use collections::FxHashMap; use gpui::{DefiniteLength, FontWeight, px, relative}; use html5ever::{ParseOpts, local_name, parse_document, tendril::TendrilSink}; use language::LanguageRegistry; use markup5ever_rcdom::RcDom; use pulldown_cmark::{Alignment, Event, Options, Parser, Tag, TagEnd}; use std::{ cell::RefCell, collections::HashMap, mem, ops::Range, path::PathBuf, rc::Rc, sync::Arc, vec, }; use ui::SharedString; pub async fn parse_markdown( markdown_input: &str, file_location_directory: Option<PathBuf>, language_registry: Option<Arc<LanguageRegistry>>, ) -> ParsedMarkdown { let mut options = Options::all(); options.remove(pulldown_cmark::Options::ENABLE_DEFINITION_LIST); let parser = Parser::new_ext(markdown_input, options); let parser = MarkdownParser::new( parser.into_offset_iter().collect(), file_location_directory, language_registry, ); let renderer = parser.parse_document().await; ParsedMarkdown { children: renderer.parsed, } } fn cleanup_html(source: &str) -> Vec<u8> { let mut writer = std::io::Cursor::new(Vec::new()); let mut reader = std::io::Cursor::new(source); let mut minify = Minifier::new( &mut writer, MinifierOptions { omit_doctype: true, collapse_whitespace: true, ..Default::default() }, ); if let Ok(()) = minify.minify(&mut reader) { writer.into_inner() } else { source.bytes().collect() } } struct MarkdownParser<'a> { tokens: Vec<(Event<'a>, Range<usize>)>, /// The current index in the tokens array cursor: usize, /// The blocks that we have successfully parsed so far parsed: Vec<ParsedMarkdownElement>, file_location_directory: Option<PathBuf>, language_registry: Option<Arc<LanguageRegistry>>, } #[derive(Debug)] struct ParseHtmlNodeContext { list_item_depth: u16, } impl Default for ParseHtmlNodeContext { fn default() -> Self { Self { list_item_depth: 1 } } } struct MarkdownListItem { content: Vec<ParsedMarkdownElement>, item_type: ParsedMarkdownListItemType, } impl Default for MarkdownListItem { fn default() -> Self { Self { content: Vec::new(), item_type: ParsedMarkdownListItemType::Unordered, } } } impl<'a> MarkdownParser<'a> { fn new( tokens: Vec<(Event<'a>, Range<usize>)>, file_location_directory: Option<PathBuf>, language_registry: Option<Arc<LanguageRegistry>>, ) -> Self { Self { tokens, file_location_directory, language_registry, cursor: 0, parsed: vec![], } } fn eof(&self) -> bool { if self.tokens.is_empty() { return true; } self.cursor >= self.tokens.len() - 1 } fn peek(&self, steps: usize) -> Option<&(Event<'_>, Range<usize>)> { if self.eof() || (steps + self.cursor) >= self.tokens.len() { return self.tokens.last(); } self.tokens.get(self.cursor + steps) } fn previous(&self) -> Option<&(Event<'_>, Range<usize>)> { if self.cursor == 0 || self.cursor > self.tokens.len() { return None; } self.tokens.get(self.cursor - 1) } fn current(&self) -> Option<&(Event<'_>, Range<usize>)> { self.peek(0) } fn current_event(&self) -> Option<&Event<'_>> { self.current().map(|(event, _)| event) } fn is_text_like(event: &Event) -> bool { match event { Event::Text(_) // Represent an inline code block | Event::Code(_) | Event::Html(_) | Event::InlineHtml(_) | Event::FootnoteReference(_) | Event::Start(Tag::Link { .. }) | Event::Start(Tag::Emphasis) | Event::Start(Tag::Strong) | Event::Start(Tag::Strikethrough) | Event::Start(Tag::Image { .. }) => { true } _ => false, } } async fn parse_document(mut self) -> Self { while !self.eof() { if let Some(block) = self.parse_block().await { self.parsed.extend(block); } else { self.cursor += 1; } } self } #[async_recursion] async fn parse_block(&mut self) -> Option<Vec<ParsedMarkdownElement>> { let (current, source_range) = self.current().unwrap(); let source_range = source_range.clone(); match current { Event::Start(tag) => match tag { Tag::Paragraph => { self.cursor += 1; let text = self.parse_text(false, Some(source_range)); Some(vec![ParsedMarkdownElement::Paragraph(text)]) } Tag::Heading { level, .. } => { let level = *level; self.cursor += 1; let heading = self.parse_heading(level); Some(vec![ParsedMarkdownElement::Heading(heading)]) } Tag::Table(alignment) => { let alignment = alignment.clone(); self.cursor += 1; let table = self.parse_table(alignment); Some(vec![ParsedMarkdownElement::Table(table)]) } Tag::List(order) => { let order = *order; self.cursor += 1; let list = self.parse_list(order).await; Some(list) } Tag::BlockQuote(_kind) => { self.cursor += 1; let block_quote = self.parse_block_quote().await; Some(vec![ParsedMarkdownElement::BlockQuote(block_quote)]) } Tag::CodeBlock(kind) => { let language = match kind { pulldown_cmark::CodeBlockKind::Indented => None, pulldown_cmark::CodeBlockKind::Fenced(language) => { if language.is_empty() { None } else { Some(language.to_string()) } } }; self.cursor += 1; let code_block = self.parse_code_block(language).await?; Some(vec![ParsedMarkdownElement::CodeBlock(code_block)]) } Tag::HtmlBlock => { self.cursor += 1; Some(self.parse_html_block().await) } _ => None, }, Event::Rule => { self.cursor += 1; Some(vec![ParsedMarkdownElement::HorizontalRule(source_range)]) } _ => None, } } fn parse_text( &mut self, should_complete_on_soft_break: bool, source_range: Option<Range<usize>>, ) -> MarkdownParagraph { let source_range = source_range.unwrap_or_else(|| { self.current() .map(|(_, range)| range.clone()) .unwrap_or_default() }); let mut markdown_text_like = Vec::new(); let mut text = String::new(); let mut bold_depth = 0; let mut italic_depth = 0; let mut strikethrough_depth = 0; let mut link: Option<Link> = None; let mut image: Option<Image> = None; let mut regions: Vec<(Range<usize>, ParsedRegion)> = vec![]; let mut highlights: Vec<(Range<usize>, MarkdownHighlight)> = vec![]; let mut link_urls: Vec<String> = vec![]; let mut link_ranges: Vec<Range<usize>> = vec![]; loop { if self.eof() { break; } let (current, _) = self.current().unwrap(); let prev_len = text.len(); match current { Event::SoftBreak => { if should_complete_on_soft_break { break; } text.push(' '); } Event::HardBreak => { text.push('\n'); } // We want to ignore any inline HTML tags in the text but keep // the text between them Event::InlineHtml(_) => {} Event::Text(t) => { text.push_str(t.as_ref()); let mut style = MarkdownHighlightStyle::default(); if bold_depth > 0 { style.weight = FontWeight::BOLD; } if italic_depth > 0 { style.italic = true; } if strikethrough_depth > 0 { style.strikethrough = true; } let last_run_len = if let Some(link) = link.clone() { regions.push(( prev_len..text.len(), ParsedRegion { code: false, link: Some(link), }, )); style.link = true; prev_len } else { // Manually scan for links let mut finder = linkify::LinkFinder::new(); finder.kinds(&[linkify::LinkKind::Url]); let mut last_link_len = prev_len; for link in finder.links(t) { let start = prev_len + link.start(); let end = prev_len + link.end(); let range = start..end; link_ranges.push(range.clone()); link_urls.push(link.as_str().to_string()); // If there is a style before we match a link, we have to add this to the highlighted ranges if style != MarkdownHighlightStyle::default() && last_link_len < start { highlights.push(( last_link_len..start, MarkdownHighlight::Style(style.clone()), )); } highlights.push(( range.clone(), MarkdownHighlight::Style(MarkdownHighlightStyle { underline: true, ..style }), )); regions.push(( range.clone(), ParsedRegion { code: false, link: Some(Link::Web { url: link.as_str().to_string(), }), }, )); last_link_len = end; } last_link_len }; if style != MarkdownHighlightStyle::default() && last_run_len < text.len() { let mut new_highlight = true; if let Some((last_range, last_style)) = highlights.last_mut() && last_range.end == last_run_len && last_style == &MarkdownHighlight::Style(style.clone()) { last_range.end = text.len(); new_highlight = false; } if new_highlight { highlights.push(( last_run_len..text.len(), MarkdownHighlight::Style(style.clone()), )); } } } Event::Code(t) => { text.push_str(t.as_ref()); let range = prev_len..text.len(); if link.is_some() { highlights.push(( range.clone(), MarkdownHighlight::Style(MarkdownHighlightStyle { link: true, ..Default::default() }), )); } regions.push(( range, ParsedRegion { code: true, link: link.clone(), }, )); } Event::Start(tag) => match tag { Tag::Emphasis => italic_depth += 1, Tag::Strong => bold_depth += 1, Tag::Strikethrough => strikethrough_depth += 1, Tag::Link { dest_url, .. } => { link = Link::identify( self.file_location_directory.clone(), dest_url.to_string(), ); } Tag::Image { dest_url, .. } => { if !text.is_empty() { let parsed_regions = MarkdownParagraphChunk::Text(ParsedMarkdownText { source_range: source_range.clone(), contents: mem::take(&mut text).into(), highlights: mem::take(&mut highlights), regions: mem::take(&mut regions), }); markdown_text_like.push(parsed_regions); } image = Image::identify( dest_url.to_string(), source_range.clone(), self.file_location_directory.clone(), ); } _ => { break; } }, Event::End(tag) => match tag { TagEnd::Emphasis => italic_depth -= 1, TagEnd::Strong => bold_depth -= 1, TagEnd::Strikethrough => strikethrough_depth -= 1, TagEnd::Link => { link = None; } TagEnd::Image => { if let Some(mut image) = image.take() { if !text.is_empty() { image.set_alt_text(std::mem::take(&mut text).into()); mem::take(&mut highlights); mem::take(&mut regions); } markdown_text_like.push(MarkdownParagraphChunk::Image(image)); } } TagEnd::Paragraph => { self.cursor += 1; break; } _ => { break; } }, _ => { break; } } self.cursor += 1; } if !text.is_empty() { markdown_text_like.push(MarkdownParagraphChunk::Text(ParsedMarkdownText { source_range, contents: text.into(), highlights, regions, })); } markdown_text_like } fn parse_heading(&mut self, level: pulldown_cmark::HeadingLevel) -> ParsedMarkdownHeading { let (_event, source_range) = self.previous().unwrap(); let source_range = source_range.clone(); let text = self.parse_text(true, None); // Advance past the heading end tag self.cursor += 1; ParsedMarkdownHeading { source_range, level: match level { pulldown_cmark::HeadingLevel::H1 => HeadingLevel::H1, pulldown_cmark::HeadingLevel::H2 => HeadingLevel::H2, pulldown_cmark::HeadingLevel::H3 => HeadingLevel::H3, pulldown_cmark::HeadingLevel::H4 => HeadingLevel::H4, pulldown_cmark::HeadingLevel::H5 => HeadingLevel::H5, pulldown_cmark::HeadingLevel::H6 => HeadingLevel::H6, }, contents: text, } } fn parse_table(&mut self, alignment: Vec<Alignment>) -> ParsedMarkdownTable { let (_event, source_range) = self.previous().unwrap(); let source_range = source_range.clone(); let mut header = vec![]; let mut body = vec![]; let mut row_columns = vec![]; let mut in_header = true; let column_alignments = alignment .iter() .map(Self::convert_alignment) .collect::<Vec<_>>(); loop { if self.eof() { break; } let (current, source_range) = self.current().unwrap(); let source_range = source_range.clone(); match current { Event::Start(Tag::TableHead) | Event::Start(Tag::TableRow) | Event::End(TagEnd::TableCell) => {