use std::cell::Cell;use std::collections::{HashMap, VecDeque};use std::rc::Rc; use anyhow::Result;use html5ever::tendril::TendrilSink;use html5ever::tree_builder::TreeBuilderOpts;use html5ever::{Attribute, ParseOpts, parse_document};use markup5ever_rcdom::{Node, NodeData, RcDom};use serde_yaml::{Mapping, Value}; use crate::markdown_parser::RUNNABLE_BLOCK_MARKDOWN_LANG;use crate::weight::CustomWeight;use crate::{    CodeBlockText, FormattedIndentTextInline, FormattedTaskList, FormattedText,    FormattedTextFragment, FormattedTextHeader, FormattedTextInline, FormattedTextLine,    FormattedTextStyles, Hyperlink, OrderedFormattedIndentTextInline,}; // Top element element tags we are not parsing for right now.// Note that we have "<b>" here because GDocs always include a top level <b> element to add additional// GDocs specific meta-data for its rich text content.const TOP_LEVEL_ELEMENT_TAGS_TO_SKIP: &[&str] = &[    "head", "body", "html", "meta", "table", "b", "div", "ul", "ol", "li", "input",];const PHRASING_ELEMENT_TAGS: &[&str] = &[    "span", "i", "code", "strong", "em", "br", "a", "s", "u", "ins",]; pub const WARP_EMBED_ATTRIBUTE_NAME: &str = "data-warp-embedded-item"; #[derive(Clone, Debug, PartialEq, Eq)]struct ListArg {    indent_level: usize,    item_type: ListType,    start_number: Rc<Cell<Option<usize>>>,} #[derive(Clone, Copy, Debug, PartialEq, Eq)]enum ListType {    Checkbox(bool),    ListItem { ordered: bool },} #[derive(Clone, Default)]struct Styling {    weight: Option<CustomWeight>,    italic: bool,    underline: bool,    strikethrough: bool,    inline_code: bool,    link: Option<String>,} impl Styling {    fn update_with_attributes(&mut self, attributes: &[Attribute]) {        for attribute in attributes {            let attribute_name = attribute.name.local.to_string();             if attribute_name == "style" {                let attribute_value = attribute.value.to_string();                let style_dict = parse_style_into_dict(attribute_value.as_str());                 if let Some(font_style) = style_dict.get("font-style") {                    self.italic = *font_style == "italic";                }                 if let Some(text_decoration) = style_dict.get("text-decoration") {                    // `text-decoration` can be used for multiple things. If we are missing line-through                    // here, don't unset the strikethrough.                    if *text_decoration == "line-through" {                        self.strikethrough = true;                    }                    if *text_decoration == "underline" {                        self.underline = true;                    }                }                 if let Some(font_weight) = style_dict.get("font-weight") {                    if *font_weight == "bold" || *font_weight == "bolder" {                        self.weight = Some(CustomWeight::Bold);                    } else if *font_weight == "normal" || *font_weight == "lighter" {                        // `lighter` is relative and we have no inherited weight to lighten from,                        // so treat it (and an explicit `normal`) as clearing any custom weight.                        self.weight = None;                    } else if let Ok(weight_value) = font_weight.parse::<i32>() {                        // Preserve the specific numeric weight (e.g. 300 -> Light, 900 -> Black)                        // instead of collapsing everything to bold-or-plain.                        self.weight = CustomWeight::from_css_numeric(weight_value);                    }                }                 // Note that there is not a definitive way in HTML to represent inline code.                // Different text editors all use different ways to represent inline code and are                // not always compatible to each other, here we chose Notion's way of representing                // inline code which is a non-transparent background. We could revisit this in the future.                if let Some(color) = style_dict.get("background") {                    self.inline_code = *color != "transparent";                }            } else if attribute_name == "href" {                let attribute_value = attribute.value.to_string();                self.link = Some(attribute_value);            }        }    }} /// Find an attribute by name, if it's present.fn get_attribute<'a>(attributes: &'a [Attribute], name: &str) -> Option<&'a str> {    attributes.iter().find_map(|attribute| {        if &attribute.name.local == name {            Some(attribute.value.as_ref())        } else {            None        }    })} fn includes_attribute(attributes: &[Attribute], name: &str) -> bool {    get_attribute(attributes, name).is_some()} fn type_matches(attributes: &[Attribute], value: &str) -> bool {    get_attribute(attributes, "type") == Some(value)} // Top-level function to parse a HTML string into a FormattedText document.pub fn parse_html(html: &str) -> Result<FormattedText> {    let opts = ParseOpts {        tree_builder: TreeBuilderOpts {            drop_doctype: true,            ..Default::default()        },        ..Default::default()    };     // Parse the document into a Dom element tree.    let dom = parse_document(RcDom::default(), opts)        .from_utf8()        .read_from(&mut html.as_bytes())?;     let mut result = VecDeque::new();     // Top-level nodes to visit.    let mut nodes: Vec<(Rc<Node>, Option<ListArg>)> = Vec::new();    nodes.push((Rc::clone(&dom.document), None));    let mut pending_inline_nodes = Vec::new();     // Active indent level decorating the current node.    let mut last_active_indent_level: Option<ListArg> = None;     while let Some((node, mut indent_level)) = nodes.pop() {        // If the indentation level has changed, we should push all pending inline nodes into the previous        // indentation level first.        if last_active_indent_level != indent_level && !pending_inline_nodes.is_empty() {            if let Some(parsed_node) =                parse_pending_inline_nodes(&pending_inline_nodes, last_active_indent_level.as_ref())            {                result.push_back(parsed_node);            }            pending_inline_nodes.clear();        }        match &node.data {            // Nodes we are not processing. Just push its children into the visit queue.            NodeData::Document            | NodeData::Doctype { .. }            | NodeData::ProcessingInstruction { .. }            | NodeData::Comment { .. } => {                for child in node.children.borrow().iter().rev() {                    nodes.push((Rc::clone(child), indent_level.clone()));                }            }            // If we observe plain text in the top level nodes. Add them as plain text lines.            NodeData::Text { contents } => {                if !contents.borrow().trim().is_empty() {                    pending_inline_nodes.push(node);                }            }            NodeData::Element { name, attrs, .. } => {                let node_name = name.local.to_string();                let mut decorated_styling = Styling::default();                 // Confluence does not follow the common pattern of marking code blocks in <pre> specifically in its view mode.                // Instead, it marks code blocks in <span> with a specific attribute.                let is_confluence_code_block = node_name.as_str() == "span"                    && includes_attribute(&attrs.borrow(), "data-ds--code--code-block");                 // If the node is an element we are skip processing, push its children into the visit queue and skip                // to the next iteration.                if TOP_LEVEL_ELEMENT_TAGS_TO_SKIP.contains(&node_name.as_str()) {                    let is_unordered_list = node_name.as_str() == "ul";                    let is_ordered_list = node_name.as_str() == "ol";                    let start_number = if is_ordered_list {                        get_attribute(&attrs.borrow(), "start").and_then(|value| value.parse().ok())                    } else {                        None                    };                    if is_unordered_list || is_ordered_list {                        let level = match &indent_level {                            Some(level) => level.indent_level + 1,                            _ => 0,                        };                        indent_level = Some(ListArg {                            item_type: ListType::ListItem {                                ordered: is_ordered_list,                            },                            start_number: Rc::new(Cell::new(start_number)),                            indent_level: level,                        });                    };                     // Check if the first node represents input.                    if let Some(NodeData::Element {                        name,                        attrs: child_node_attr,                        ..                    }) = node.children.borrow().iter().next().map(|node| &node.data)                    {                        let child_node_name = name.local.to_string();                        if let Some(indent_arg) = &mut indent_level                            && child_node_name.as_str() == "input"                            && type_matches(&child_node_attr.borrow(), "checkbox")                        {                            indent_arg.item_type = ListType::Checkbox(includes_attribute(                                &child_node_attr.borrow(),                                "checked",                            ));                        }                    }                     for child in node.children.borrow().iter().rev() {                        nodes.push((Rc::clone(child), indent_level.clone()));                    }                     if !pending_inline_nodes.is_empty() {                        if let Some(parsed_nodes) = parse_pending_inline_nodes(                            &pending_inline_nodes,                            last_active_indent_level.as_ref(),                        ) {                            result.push_back(parsed_nodes);                        }                        pending_inline_nodes.clear();                    }                     last_active_indent_level = indent_level;                    continue;                } else if PHRASING_ELEMENT_TAGS.contains(&node_name.as_str())                    && !is_confluence_code_block                {                    pending_inline_nodes.push(node);                    last_active_indent_level = indent_level;                    continue;                }                 if !pending_inline_nodes.is_empty() {                    if let Some(parsed_nodes) = parse_pending_inline_nodes(                        &pending_inline_nodes,                        last_active_indent_level.as_ref(),                    ) {                        result.push_back(parsed_nodes);                    }                    pending_inline_nodes.clear();                }                 // Update styling based on the node's attribute.                decorated_styling.update_with_attributes(&attrs.borrow());                 result.push_back(match node_name.as_str() {                    // If it's a code block, process its children node as plain text.                    "pre" => {                        if let Some(val) = get_attribute(&attrs.borrow(), WARP_EMBED_ATTRIBUTE_NAME)                        {                            FormattedTextLine::Embedded(Mapping::from_iter([(                                Value::String("id".to_string()),                                Value::String(val.to_string()),                            )]))                        } else {                            // TODO: Support Github's code block representation.                            let (content, language) =                                parse_code_block_and_language(&node.children.borrow());                            FormattedTextLine::CodeBlock(CodeBlockText {                                lang: language.unwrap_or(RUNNABLE_BLOCK_MARKDOWN_LANG.to_string()),                                code: content,                            })                        }                    }                    "span" if is_confluence_code_block => {                        FormattedTextLine::CodeBlock(CodeBlockText {                            lang: get_attribute(&attrs.borrow(), "data-code-lang")                                .unwrap_or(RUNNABLE_BLOCK_MARKDOWN_LANG)                                .to_string(),                            code: parse_text_only(&node.children.borrow()),                        })                    }                    "h1" => FormattedTextLine::Heading(FormattedTextHeader {                        heading_size: 1,                        text: parse_phrasing_content(                            &node.children.borrow(),                            decorated_styling.clone(),                        ),                    }),                    "h2" => FormattedTextLine::Heading(FormattedTextHeader {                        heading_size: 2,                        text: parse_phrasing_content(                            &node.children.borrow(),                            decorated_styling.clone(),                        ),                    }),                    "h3" => FormattedTextLine::Heading(FormattedTextHeader {                        heading_size: 3,                        text: parse_phrasing_content(                            &node.children.borrow(),                            decorated_styling.clone(),                        ),                    }),                    "h4" => FormattedTextLine::Heading(FormattedTextHeader {                        heading_size: 4,                        text: parse_phrasing_content(                            &node.children.borrow(),                            decorated_styling.clone(),                        ),                    }),                    "h5" => FormattedTextLine::Heading(FormattedTextHeader {                        heading_size: 5,                        text: parse_phrasing_content(                            &node.children.borrow(),                            decorated_styling.clone(),                        ),                    }),                    "h6" => FormattedTextLine::Heading(FormattedTextHeader {                        heading_size: 6,                        text: parse_phrasing_content(                            &node.children.borrow(),                            decorated_styling.clone(),                        ),                    }),                    "br" => FormattedTextLine::LineBreak,                    "hr" => FormattedTextLine::HorizontalRule,                    _ => {                        // Take into consideration the indent level when parsing the nodes.                        let parsed_node = parse_pending_inline_nodes(                            &node.children.borrow(),                            indent_level.as_ref(),                        );                         match parsed_node {                            Some(node) => node,                            None => FormattedTextLine::Line(Vec::new()),                        }                    }                })            }        }         last_active_indent_level = indent_level;    }     if !pending_inline_nodes.is_empty() {        if let Some(parsed_nodes) =            parse_pending_inline_nodes(&pending_inline_nodes, last_active_indent_level.as_ref())        {            result.push_back(parsed_nodes);        }        pending_inline_nodes.clear();    }     Ok(FormattedText { lines: result })} // Push all pending inline nodes into the result. Take into consideration the active indent level.fn parse_pending_inline_nodes(    nodes: &[Rc<Node>],    last_active_indent_level: Option<&ListArg>,) -> Option<FormattedTextLine> {    let internal = parse_phrasing_content(nodes, Default::default());     if !internal.is_empty() {        Some(match last_active_indent_level {            Some(list) => match list.item_type {                ListType::ListItem { ordered: true } => {                    FormattedTextLine::OrderedList(OrderedFormattedIndentTextInline {                        // Take the start number, so that it's only applied to the first item in                        // the list.                        number: list.start_number.take(),                        indented_text: FormattedIndentTextInline {                            indent_level: list.indent_level,                            text: internal,                        },                    })                }                ListType::ListItem { ordered: false } => {                    FormattedTextLine::UnorderedList(FormattedIndentTextInline {                        indent_level: list.indent_level,                        text: internal,                    })                }                ListType::Checkbox(checked) => FormattedTextLine::TaskList(FormattedTaskList {                    complete: checked,                    indent_level: list.indent_level,                    text: internal,                }),            },            None => FormattedTextLine::Line(internal),        })    } else {        None    }} // Parse the phrasing content: https://developer.mozilla.org/en-US/docs/Web/HTML/Content_categories#phrasing_content// into an inline formatted text.fn parse_phrasing_content(nodes: &[Rc<Node>], text_styling: Styling) -> FormattedTextInline {    let mut result = Vec::new();    for node in nodes {        if is_spacing_span(node) {            result.push(phrasing_to_formatted_text(" ", &text_styling));            continue;        }         match &node.data {            // We should not observe these in the phrasing content.            NodeData::Document            | NodeData::Doctype { .. }            | NodeData::ProcessingInstruction { .. }            | NodeData::Comment { .. } => {}            // Push text fragment based on the current text_styling.            NodeData::Text { contents } => {                let content = contents.borrow().trim_end_matches('\n').to_string();                 // Filter out singular empty lines after trimming as they are from HTML formatting and shouldn't                // be inserted into the content. The rare valid case we might be missing here is <p>\n</p>.                // But all major rich text editors I have tested use <br> to represent it instead.                if content.is_empty() {                    continue;                }                 result.push(phrasing_to_formatted_text(content, &text_styling));            }            NodeData::Element { name, attrs, .. } => {                let node_name = name.local.to_string();                let mut decorated_styling = text_styling.clone();                decorated_styling.update_with_attributes(&attrs.borrow());                match node_name.as_ref() {                    "b" | "strong" => decorated_styling.weight = Some(CustomWeight::Bold),                    "i" | "em" => decorated_styling.italic = true,                    "s" => decorated_styling.strikethrough = true,                    "u" | "ins" => decorated_styling.underline = true,                    "code" => decorated_styling.inline_code = true,                    // TODO: We need to add more phrasing styling we support (e.g. links) here.                    // https://linear.app/warpdotdev/issue/CLD-335/add-html-parsing-for-headers-and-lists                    _ => (),                };                 result.extend(parse_phrasing_content(                    node.children.borrow().as_ref(),                    decorated_styling,                ));            }        }    }    result} /// Converts styled phrasing text to a fragment of formatted text.fn phrasing_to_formatted_text(text: impl Into<String>, styling: &Styling) -> FormattedTextFragment {    FormattedTextFragment {        text: text.into(),        styles: FormattedTextStyles {            weight: styling.weight,            italic: styling.italic,            underline: styling.underline,            strikethrough: styling.strikethrough,            hyperlink: styling.link.clone().map(Hyperlink::Url),            inline_code: styling.inline_code,        },    }} /// Chrome and Safari replace some spaces in copied content with non-breaking spaces. This matches/// such spaces.////// See [this ProseMirror thread](https://discuss.prosemirror.net/t/non-breaking-spaces-being-added-to-pasted-html/3911/4).fn is_spacing_span(node: &Rc<Node>) -> bool {    if let NodeData::Element { name, attrs, .. } = &node.data {        if &name.local != "span" {            return false;        }         // The span must have either no attrs or a single `class="Apple-converted-space"` class.        let attrs = attrs.borrow();        if attrs.len() > 1 {            return false;        }        let css_class = attrs.first().filter(|attr| &attr.name.local == "class");        if css_class.is_some_and(|class| &*class.value != "Apple-converted-space") {            return false;        }         let content = node.children.borrow();        if content.len() != 1 {

Lines 1–500 of 575. Load more