use crate::markdown_elements::*;use async_recursion::async_recursion;use collections::FxHashMap;use gpui::FontWeight;use language::LanguageRegistry;use pulldown_cmark::{Alignment, Event, Options, Parser, Tag, TagEnd};use std::{ops::Range, path::PathBuf, sync::Arc}; pub async fn parse_markdown( markdown_input: &str, file_location_directory: Option<PathBuf>, language_registry: Option<Arc<LanguageRegistry>>,) -> ParsedMarkdown { let mut options = Options::all(); options.remove(pulldown_cmark::Options::ENABLE_DEFINITION_LIST); let parser = Parser::new_ext(markdown_input, options); let parser = MarkdownParser::new( parser.into_offset_iter().collect(), file_location_directory, language_registry, ); let renderer = parser.parse_document().await; ParsedMarkdown { children: renderer.parsed, }} struct MarkdownParser<'a> { tokens: Vec<(Event<'a>, Range<usize>)>, /// The current index in the tokens array cursor: usize, /// The blocks that we have successfully parsed so far parsed: Vec<ParsedMarkdownElement>, file_location_directory: Option<PathBuf>, language_registry: Option<Arc<LanguageRegistry>>,} impl<'a> MarkdownParser<'a> { fn new( tokens: Vec<(Event<'a>, Range<usize>)>, file_location_directory: Option<PathBuf>, language_registry: Option<Arc<LanguageRegistry>>, ) -> Self { Self { tokens, file_location_directory, language_registry, cursor: 0, parsed: vec![], } } fn eof(&self) -> bool { if self.tokens.is_empty() { return true; } self.cursor >= self.tokens.len() - 1 } fn peek(&self, steps: usize) -> Option<&(Event, Range<usize>)> { if self.eof() || (steps + self.cursor) >= self.tokens.len() { return self.tokens.last(); } return self.tokens.get(self.cursor + steps); } fn previous(&self) -> Option<&(Event, Range<usize>)> { if self.cursor == 0 || self.cursor > self.tokens.len() { return None; } return self.tokens.get(self.cursor - 1); } fn current(&self) -> Option<&(Event, Range<usize>)> { return self.peek(0); } fn current_event(&self) -> Option<&Event> { return self.current().map(|(event, _)| event); } fn is_text_like(event: &Event) -> bool { match event { Event::Text(_) // Represent an inline code block | Event::Code(_) | Event::Html(_) | Event::FootnoteReference(_) | Event::Start(Tag::Link { link_type: _, dest_url: _, title: _, id: _ }) | Event::Start(Tag::Emphasis) | Event::Start(Tag::Strong) | Event::Start(Tag::Strikethrough) | Event::Start(Tag::Image { link_type: _, dest_url: _, title: _, id: _ }) => { true } _ => false, } } async fn parse_document(mut self) -> Self { while !self.eof() { if let Some(block) = self.parse_block().await { self.parsed.extend(block); } } self } #[async_recursion] async fn parse_block(&mut self) -> Option<Vec<ParsedMarkdownElement>> { let (current, source_range) = self.current().unwrap(); let source_range = source_range.clone(); match current { Event::Start(tag) => match tag { Tag::Paragraph => { self.cursor += 1; let text = self.parse_text(false, Some(source_range)); Some(vec![ParsedMarkdownElement::Paragraph(text)]) } Tag::Heading { level, id: _, classes: _, attrs: _, } => { let level = *level; self.cursor += 1; let heading = self.parse_heading(level); Some(vec![ParsedMarkdownElement::Heading(heading)]) } Tag::Table(alignment) => { let alignment = alignment.clone(); self.cursor += 1; let table = self.parse_table(alignment); Some(vec![ParsedMarkdownElement::Table(table)]) } Tag::List(order) => { let order = *order; self.cursor += 1; let list = self.parse_list(order).await; Some(list) } Tag::BlockQuote(_kind) => { self.cursor += 1; let block_quote = self.parse_block_quote().await; Some(vec![ParsedMarkdownElement::BlockQuote(block_quote)]) } Tag::CodeBlock(kind) => { let language = match kind { pulldown_cmark::CodeBlockKind::Indented => None, pulldown_cmark::CodeBlockKind::Fenced(language) => { if language.is_empty() { None } else { Some(language.to_string()) } } }; self.cursor += 1; let code_block = self.parse_code_block(language).await; Some(vec![ParsedMarkdownElement::CodeBlock(code_block)]) } _ => { self.cursor += 1; None } }, Event::Rule => { let source_range = source_range.clone(); self.cursor += 1; Some(vec![ParsedMarkdownElement::HorizontalRule(source_range)]) } _ => { self.cursor += 1; None } } } fn parse_text( &mut self, should_complete_on_soft_break: bool, source_range: Option<Range<usize>>, ) -> ParsedMarkdownText { let source_range = source_range.unwrap_or_else(|| { self.current() .map(|(_, range)| range.clone()) .unwrap_or_default() }); let mut text = String::new(); let mut bold_depth = 0; let mut italic_depth = 0; let mut strikethrough_depth = 0; let mut link: Option<Link> = None; let mut region_ranges: Vec<Range<usize>> = vec![]; let mut regions: Vec<ParsedRegion> = vec![]; let mut highlights: Vec<(Range<usize>, MarkdownHighlight)> = vec![]; let mut link_urls: Vec<String> = vec![]; let mut link_ranges: Vec<Range<usize>> = vec![]; loop { if self.eof() { break; } let (current, _source_range) = self.current().unwrap(); let prev_len = text.len(); match current { Event::SoftBreak => { if should_complete_on_soft_break { break; } // `Some text\nSome more text` should be treated as a single line. text.push(' '); } Event::HardBreak => { text.push('\n'); } Event::Text(t) => { text.push_str(t.as_ref()); let mut style = MarkdownHighlightStyle::default(); if bold_depth > 0 { style.weight = FontWeight::BOLD; } if italic_depth > 0 { style.italic = true; } if strikethrough_depth > 0 { style.strikethrough = true; } let last_run_len = if let Some(link) = link.clone() { region_ranges.push(prev_len..text.len()); regions.push(ParsedRegion { code: false, link: Some(link), }); style.underline = true; prev_len } else { // Manually scan for links let mut finder = linkify::LinkFinder::new(); finder.kinds(&[linkify::LinkKind::Url]); let mut last_link_len = prev_len; for link in finder.links(t) { let start = link.start(); let end = link.end(); let range = (prev_len + start)..(prev_len + end); link_ranges.push(range.clone()); link_urls.push(link.as_str().to_string()); // If there is a style before we match a link, we have to add this to the highlighted ranges if style != MarkdownHighlightStyle::default() && last_link_len < link.start() { highlights.push(( last_link_len..link.start(), MarkdownHighlight::Style(style.clone()), )); } highlights.push(( range.clone(), MarkdownHighlight::Style(MarkdownHighlightStyle { underline: true, ..style }), )); region_ranges.push(range.clone()); regions.push(ParsedRegion { code: false, link: Some(Link::Web { url: link.as_str().to_string(), }), }); last_link_len = end; } last_link_len }; if style != MarkdownHighlightStyle::default() && last_run_len < text.len() { let mut new_highlight = true; if let Some((last_range, last_style)) = highlights.last_mut() { if last_range.end == last_run_len && last_style == &MarkdownHighlight::Style(style.clone()) { last_range.end = text.len(); new_highlight = false; } } if new_highlight { highlights .push((last_run_len..text.len(), MarkdownHighlight::Style(style))); } } } // Note: This event means "inline code" and not "code block" Event::Code(t) => { text.push_str(t.as_ref()); region_ranges.push(prev_len..text.len()); if link.is_some() { highlights.push(( prev_len..text.len(), MarkdownHighlight::Style(MarkdownHighlightStyle { underline: true, ..Default::default() }), )); } regions.push(ParsedRegion { code: true, link: link.clone(), }); } Event::Start(tag) => match tag { Tag::Emphasis => italic_depth += 1, Tag::Strong => bold_depth += 1, Tag::Strikethrough => strikethrough_depth += 1, Tag::Link { link_type: _, dest_url, title: _, id: _, } => { link = Link::identify( self.file_location_directory.clone(), dest_url.to_string(), ); } _ => { break; } }, Event::End(tag) => match tag { TagEnd::Emphasis => { italic_depth -= 1; } TagEnd::Strong => { bold_depth -= 1; } TagEnd::Strikethrough => { strikethrough_depth -= 1; } TagEnd::Link => { link = None; } TagEnd::Paragraph => { self.cursor += 1; break; } _ => { break; } }, _ => { break; } } self.cursor += 1; } ParsedMarkdownText { source_range, contents: text, highlights, regions, region_ranges, } } fn parse_heading(&mut self, level: pulldown_cmark::HeadingLevel) -> ParsedMarkdownHeading { let (_event, source_range) = self.previous().unwrap(); let source_range = source_range.clone(); let text = self.parse_text(true, None); // Advance past the heading end tag self.cursor += 1; ParsedMarkdownHeading { source_range: source_range.clone(), level: match level { pulldown_cmark::HeadingLevel::H1 => HeadingLevel::H1, pulldown_cmark::HeadingLevel::H2 => HeadingLevel::H2, pulldown_cmark::HeadingLevel::H3 => HeadingLevel::H3, pulldown_cmark::HeadingLevel::H4 => HeadingLevel::H4, pulldown_cmark::HeadingLevel::H5 => HeadingLevel::H5, pulldown_cmark::HeadingLevel::H6 => HeadingLevel::H6, }, contents: text, } } fn parse_table(&mut self, alignment: Vec<Alignment>) -> ParsedMarkdownTable { let (_event, source_range) = self.previous().unwrap(); let source_range = source_range.clone(); let mut header = ParsedMarkdownTableRow::new(); let mut body = vec![]; let mut current_row = vec![]; let mut in_header = true; let column_alignments = alignment.iter().map(Self::convert_alignment).collect(); loop { if self.eof() { break; } let (current, source_range) = self.current().unwrap(); let source_range = source_range.clone(); match current { Event::Start(Tag::TableHead) | Event::Start(Tag::TableRow) | Event::End(TagEnd::TableCell) => { self.cursor += 1; } Event::Start(Tag::TableCell) => { self.cursor += 1; let cell_contents = self.parse_text(false, Some(source_range)); current_row.push(cell_contents); } Event::End(TagEnd::TableHead) | Event::End(TagEnd::TableRow) => { self.cursor += 1; let new_row = std::mem::take(&mut current_row); if in_header { header.children = new_row; in_header = false; } else { let row = ParsedMarkdownTableRow::with_children(new_row); body.push(row); } } Event::End(TagEnd::Table) => { self.cursor += 1; break; } _ => { break; } } } ParsedMarkdownTable { source_range, header, body, column_alignments, } } fn convert_alignment(alignment: &Alignment) -> ParsedMarkdownTableAlignment { match alignment { Alignment::None => ParsedMarkdownTableAlignment::None, Alignment::Left => ParsedMarkdownTableAlignment::Left, Alignment::Center => ParsedMarkdownTableAlignment::Center, Alignment::Right => ParsedMarkdownTableAlignment::Right, } } async fn parse_list(&mut self, order: Option<u64>) -> Vec<ParsedMarkdownElement> { let (_, list_source_range) = self.previous().unwrap(); let mut items = Vec::new(); let mut items_stack = vec![Vec::new()]; let mut depth = 1; let mut task_item = None; let mut order = order; let mut order_stack = Vec::new(); let mut insertion_indices = FxHashMap::default(); let mut source_ranges = FxHashMap::default(); let mut start_item_range = list_source_range.clone(); while !self.eof() { let (current, source_range) = self.current().unwrap(); match current { Event::Start(Tag::List(new_order)) => { if items_stack.last().is_some() && !insertion_indices.contains_key(&depth) { insertion_indices.insert(depth, items.len()); } // We will use the start of the nested list as the end for the current item's range,Lines 1–500 of 1347. Load more
use crate::markdown_elements::*; use async_recursion::async_recursion; use collections::FxHashMap; use gpui::FontWeight; use language::LanguageRegistry; use pulldown_cmark::{Alignment, Event, Options, Parser, Tag, TagEnd}; use std::{ops::Range, path::PathBuf, sync::Arc}; pub async fn parse_markdown( markdown_input: &str, file_location_directory: Option<PathBuf>, language_registry: Option<Arc<LanguageRegistry>>, ) -> ParsedMarkdown { let mut options = Options::all(); options.remove(pulldown_cmark::Options::ENABLE_DEFINITION_LIST); let parser = Parser::new_ext(markdown_input, options); let parser = MarkdownParser::new( parser.into_offset_iter().collect(), file_location_directory, language_registry, ); let renderer = parser.parse_document().await; ParsedMarkdown { children: renderer.parsed, } } struct MarkdownParser<'a> { tokens: Vec<(Event<'a>, Range<usize>)>, /// The current index in the tokens array cursor: usize, /// The blocks that we have successfully parsed so far parsed: Vec<ParsedMarkdownElement>, file_location_directory: Option<PathBuf>, language_registry: Option<Arc<LanguageRegistry>>, } impl<'a> MarkdownParser<'a> { fn new( tokens: Vec<(Event<'a>, Range<usize>)>, file_location_directory: Option<PathBuf>, language_registry: Option<Arc<LanguageRegistry>>, ) -> Self { Self { tokens, file_location_directory, language_registry, cursor: 0, parsed: vec![], } } fn eof(&self) -> bool { if self.tokens.is_empty() { return true; } self.cursor >= self.tokens.len() - 1 } fn peek(&self, steps: usize) -> Option<&(Event, Range<usize>)> { if self.eof() || (steps + self.cursor) >= self.tokens.len() { return self.tokens.last(); } return self.tokens.get(self.cursor + steps); } fn previous(&self) -> Option<&(Event, Range<usize>)> { if self.cursor == 0 || self.cursor > self.tokens.len() { return None; } return self.tokens.get(self.cursor - 1); } fn current(&self) -> Option<&(Event, Range<usize>)> { return self.peek(0); } fn current_event(&self) -> Option<&Event> { return self.current().map(|(event, _)| event); } fn is_text_like(event: &Event) -> bool { match event { Event::Text(_) // Represent an inline code block | Event::Code(_) | Event::Html(_) | Event::FootnoteReference(_) | Event::Start(Tag::Link { link_type: _, dest_url: _, title: _, id: _ }) | Event::Start(Tag::Emphasis) | Event::Start(Tag::Strong) | Event::Start(Tag::Strikethrough) | Event::Start(Tag::Image { link_type: _, dest_url: _, title: _, id: _ }) => { true } _ => false, } } async fn parse_document(mut self) -> Self { while !self.eof() { if let Some(block) = self.parse_block().await { self.parsed.extend(block); } } self } #[async_recursion] async fn parse_block(&mut self) -> Option<Vec<ParsedMarkdownElement>> { let (current, source_range) = self.current().unwrap(); let source_range = source_range.clone(); match current { Event::Start(tag) => match tag { Tag::Paragraph => { self.cursor += 1; let text = self.parse_text(false, Some(source_range)); Some(vec![ParsedMarkdownElement::Paragraph(text)]) } Tag::Heading { level, id: _, classes: _, attrs: _, } => { let level = *level; self.cursor += 1; let heading = self.parse_heading(level); Some(vec![ParsedMarkdownElement::Heading(heading)]) } Tag::Table(alignment) => { let alignment = alignment.clone(); self.cursor += 1; let table = self.parse_table(alignment); Some(vec![ParsedMarkdownElement::Table(table)]) } Tag::List(order) => { let order = *order; self.cursor += 1; let list = self.parse_list(order).await; Some(list) } Tag::BlockQuote(_kind) => { self.cursor += 1; let block_quote = self.parse_block_quote().await; Some(vec![ParsedMarkdownElement::BlockQuote(block_quote)]) } Tag::CodeBlock(kind) => { let language = match kind { pulldown_cmark::CodeBlockKind::Indented => None, pulldown_cmark::CodeBlockKind::Fenced(language) => { if language.is_empty() { None } else { Some(language.to_string()) } } }; self.cursor += 1; let code_block = self.parse_code_block(language).await; Some(vec![ParsedMarkdownElement::CodeBlock(code_block)]) } _ => { self.cursor += 1; None } }, Event::Rule => { let source_range = source_range.clone(); self.cursor += 1; Some(vec![ParsedMarkdownElement::HorizontalRule(source_range)]) } _ => { self.cursor += 1; None } } } fn parse_text( &mut self, should_complete_on_soft_break: bool, source_range: Option<Range<usize>>, ) -> ParsedMarkdownText { let source_range = source_range.unwrap_or_else(|| { self.current() .map(|(_, range)| range.clone()) .unwrap_or_default() }); let mut text = String::new(); let mut bold_depth = 0; let mut italic_depth = 0; let mut strikethrough_depth = 0; let mut link: Option<Link> = None; let mut region_ranges: Vec<Range<usize>> = vec![]; let mut regions: Vec<ParsedRegion> = vec![]; let mut highlights: Vec<(Range<usize>, MarkdownHighlight)> = vec![]; let mut link_urls: Vec<String> = vec![]; let mut link_ranges: Vec<Range<usize>> = vec![]; loop { if self.eof() { break; } let (current, _source_range) = self.current().unwrap(); let prev_len = text.len(); match current { Event::SoftBreak => { if should_complete_on_soft_break { break; } // `Some text\nSome more text` should be treated as a single line. text.push(' '); } Event::HardBreak => { text.push('\n'); } Event::Text(t) => { text.push_str(t.as_ref()); let mut style = MarkdownHighlightStyle::default(); if bold_depth > 0 { style.weight = FontWeight::BOLD; } if italic_depth > 0 { style.italic = true; } if strikethrough_depth > 0 { style.strikethrough = true; } let last_run_len = if let Some(link) = link.clone() { region_ranges.push(prev_len..text.len()); regions.push(ParsedRegion { code: false, link: Some(link), }); style.underline = true; prev_len } else { // Manually scan for links let mut finder = linkify::LinkFinder::new(); finder.kinds(&[linkify::LinkKind::Url]); let mut last_link_len = prev_len; for link in finder.links(t) { let start = link.start(); let end = link.end(); let range = (prev_len + start)..(prev_len + end); link_ranges.push(range.clone()); link_urls.push(link.as_str().to_string()); // If there is a style before we match a link, we have to add this to the highlighted ranges if style != MarkdownHighlightStyle::default() && last_link_len < link.start() { highlights.push(( last_link_len..link.start(), MarkdownHighlight::Style(style.clone()), )); } highlights.push(( range.clone(), MarkdownHighlight::Style(MarkdownHighlightStyle { underline: true, ..style }), )); region_ranges.push(range.clone()); regions.push(ParsedRegion { code: false, link: Some(Link::Web { url: link.as_str().to_string(), }), }); last_link_len = end; } last_link_len }; if style != MarkdownHighlightStyle::default() && last_run_len < text.len() { let mut new_highlight = true; if let Some((last_range, last_style)) = highlights.last_mut() { if last_range.end == last_run_len && last_style == &MarkdownHighlight::Style(style.clone()) { last_range.end = text.len(); new_highlight = false; } } if new_highlight { highlights .push((last_run_len..text.len(), MarkdownHighlight::Style(style))); } } } // Note: This event means "inline code" and not "code block" Event::Code(t) => { text.push_str(t.as_ref()); region_ranges.push(prev_len..text.len()); if link.is_some() { highlights.push(( prev_len..text.len(), MarkdownHighlight::Style(MarkdownHighlightStyle { underline: true, ..Default::default() }), )); } regions.push(ParsedRegion { code: true, link: link.clone(), }); } Event::Start(tag) => match tag { Tag::Emphasis => italic_depth += 1, Tag::Strong => bold_depth += 1, Tag::Strikethrough => strikethrough_depth += 1, Tag::Link { link_type: _, dest_url, title: _, id: _, } => { link = Link::identify( self.file_location_directory.clone(), dest_url.to_string(), ); } _ => { break; } }, Event::End(tag) => match tag { TagEnd::Emphasis => { italic_depth -= 1; } TagEnd::Strong => { bold_depth -= 1; } TagEnd::Strikethrough => { strikethrough_depth -= 1; } TagEnd::Link => { link = None; } TagEnd::Paragraph => { self.cursor += 1; break; } _ => { break; } }, _ => { break; } } self.cursor += 1; } ParsedMarkdownText { source_range, contents: text, highlights, regions, region_ranges, } } fn parse_heading(&mut self, level: pulldown_cmark::HeadingLevel) -> ParsedMarkdownHeading { let (_event, source_range) = self.previous().unwrap(); let source_range = source_range.clone(); let text = self.parse_text(true, None); // Advance past the heading end tag self.cursor += 1; ParsedMarkdownHeading { source_range: source_range.clone(), level: match level { pulldown_cmark::HeadingLevel::H1 => HeadingLevel::H1, pulldown_cmark::HeadingLevel::H2 => HeadingLevel::H2, pulldown_cmark::HeadingLevel::H3 => HeadingLevel::H3, pulldown_cmark::HeadingLevel::H4 => HeadingLevel::H4, pulldown_cmark::HeadingLevel::H5 => HeadingLevel::H5, pulldown_cmark::HeadingLevel::H6 => HeadingLevel::H6, }, contents: text, } } fn parse_table(&mut self, alignment: Vec<Alignment>) -> ParsedMarkdownTable { let (_event, source_range) = self.previous().unwrap(); let source_range = source_range.clone(); let mut header = ParsedMarkdownTableRow::new(); let mut body = vec![]; let mut current_row = vec![]; let mut in_header = true; let column_alignments = alignment.iter().map(Self::convert_alignment).collect(); loop { if self.eof() { break; } let (current, source_range) = self.current().unwrap(); let source_range = source_range.clone(); match current { Event::Start(Tag::TableHead) | Event::Start(Tag::TableRow) | Event::End(TagEnd::TableCell) => { self.cursor += 1; } Event::Start(Tag::TableCell) => { self.cursor += 1; let cell_contents = self.parse_text(false, Some(source_range)); current_row.push(cell_contents); } Event::End(TagEnd::TableHead) | Event::End(TagEnd::TableRow) => { self.cursor += 1; let new_row = std::mem::take(&mut current_row); if in_header { header.children = new_row; in_header = false; } else { let row = ParsedMarkdownTableRow::with_children(new_row); body.push(row); } } Event::End(TagEnd::Table) => { self.cursor += 1; break; } _ => { break; } } } ParsedMarkdownTable { source_range, header, body, column_alignments, } } fn convert_alignment(alignment: &Alignment) -> ParsedMarkdownTableAlignment { match alignment { Alignment::None => ParsedMarkdownTableAlignment::None, Alignment::Left => ParsedMarkdownTableAlignment::Left, Alignment::Center => ParsedMarkdownTableAlignment::Center, Alignment::Right => ParsedMarkdownTableAlignment::Right, } } async fn parse_list(&mut self, order: Option<u64>) -> Vec<ParsedMarkdownElement> { let (_, list_source_range) = self.previous().unwrap(); let mut items = Vec::new(); let mut items_stack = vec![Vec::new()]; let mut depth = 1; let mut task_item = None; let mut order = order; let mut order_stack = Vec::new(); let mut insertion_indices = FxHashMap::default(); let mut source_ranges = FxHashMap::default(); let mut start_item_range = list_source_range.clone(); while !self.eof() { let (current, source_range) = self.current().unwrap(); match current { Event::Start(Tag::List(new_order)) => { if items_stack.last().is_some() && !insertion_indices.contains_key(&depth) { insertion_indices.insert(depth, items.len()); } // We will use the start of the nested list as the end for the current item's range,