use crate::{OffsetUtf16, Point, PointUtf16, TextSummary, Unclipped};use heapless::String as ArrayString;use std::{cmp, ops::Range};use sum_tree::Bias;use unicode_segmentation::GraphemeCursor;use util::debug_panic; #[cfg(not(all(test, not(rust_analyzer))))]pub(crate) type Bitmap = u128;#[cfg(all(test, not(rust_analyzer)))]pub(crate) type Bitmap = u16; pub(crate) const MIN_BASE: usize = MAX_BASE / 2;pub(crate) const MAX_BASE: usize = Bitmap::BITS as usize; #[derive(Clone, Debug, Default)]pub struct Chunk {    /// If bit[i] is set, then the character at index i is the start of a UTF-8 character in the    /// text.    chars: Bitmap,    /// The number of set bits is the number of UTF-16 code units it would take to represent the    /// text.    ///    /// Bit[i] is set if text[i] is the start of a UTF-8 character. If the character would    /// take two UTF-16 code units, then bit[i+1] is also set. (Rust chars never take more    /// than two UTF-16 code units.)    chars_utf16: Bitmap,    /// If bit[i] is set, then the character at index i is an ascii newline.    newlines: Bitmap,    /// If bit[i] is set, then the character at index i is an ascii tab.    tabs: Bitmap,    pub text: ArrayString<MAX_BASE, u8>,} #[inline(always)]const fn saturating_shl_mask(offset: u32) -> Bitmap {    (1 as Bitmap).unbounded_shl(offset).wrapping_sub(1)} #[inline(always)]const fn saturating_shr_mask(offset: u32) -> Bitmap {    !Bitmap::MAX.unbounded_shr(offset)} impl Chunk {    pub const MASK_BITS: usize = Bitmap::BITS as usize;     #[inline(always)]    pub fn new(text: &str) -> Self {        let text = {            let mut buf = ArrayString::new();            buf.push_str(text).unwrap();            buf        };         const CHUNK_SIZE: usize = 8;         let mut chars_bytes = [0; MAX_BASE / CHUNK_SIZE];        let mut newlines_bytes = [0; MAX_BASE / CHUNK_SIZE];        let mut tabs_bytes = [0; MAX_BASE / CHUNK_SIZE];        let mut chars_utf16_bytes = [0; MAX_BASE / CHUNK_SIZE];         let mut chunk_ix = 0;         let mut bytes = text.as_bytes();        while !bytes.is_empty() {            let (chunk, rest) = bytes.split_at(bytes.len().min(CHUNK_SIZE));            bytes = rest;             let mut chars = 0;            let mut newlines = 0;            let mut tabs = 0;            let mut chars_utf16 = 0;             for (ix, &b) in chunk.iter().enumerate() {                chars |= (util::is_utf8_char_boundary(b) as u8) << ix;                newlines |= ((b == b'\n') as u8) << ix;                tabs |= ((b == b'\t') as u8) << ix;                // b >= 240 when we are at the first byte of the 4 byte encoded                // utf-8 code point (U+010000 or greater) it means that it would                // be encoded as two 16-bit code units in utf-16                chars_utf16 |= ((b >= 240) as u8) << ix;            }             chars_bytes[chunk_ix] = chars;            newlines_bytes[chunk_ix] = newlines;            tabs_bytes[chunk_ix] = tabs;            chars_utf16_bytes[chunk_ix] = chars_utf16;             chunk_ix += 1;        }         let chars = Bitmap::from_le_bytes(chars_bytes);         Chunk {            text,            chars,            chars_utf16: (Bitmap::from_le_bytes(chars_utf16_bytes) << 1) | chars,            newlines: Bitmap::from_le_bytes(newlines_bytes),            tabs: Bitmap::from_le_bytes(tabs_bytes),        }    }     #[inline(always)]    pub fn push_str(&mut self, text: &str) {        self.append(Chunk::new(text).as_slice());    }     #[inline(always)]    pub fn prepend_str(&mut self, text: &str) {        self.prepend(Chunk::new(text).as_slice());    }     #[inline(always)]    pub fn append(&mut self, slice: ChunkSlice) {        if slice.is_empty() {            return;        };         let base_ix = self.text.len();        self.chars |= slice.chars << base_ix;        self.chars_utf16 |= slice.chars_utf16 << base_ix;        self.newlines |= slice.newlines << base_ix;        self.tabs |= slice.tabs << base_ix;        self.text.push_str(slice.text).unwrap();    }     #[inline(always)]    pub fn prepend(&mut self, slice: ChunkSlice) {        if slice.is_empty() {            return;        }        if self.text.is_empty() {            *self = Chunk::new(slice.text);            return;        }         let shift = slice.text.len();        self.chars = slice.chars | (self.chars << shift);        self.chars_utf16 = slice.chars_utf16 | (self.chars_utf16 << shift);        self.newlines = slice.newlines | (self.newlines << shift);        self.tabs = slice.tabs | (self.tabs << shift);         let mut new_text = ArrayString::<MAX_BASE, u8>::new();        new_text.push_str(slice.text).unwrap();        new_text.push_str(&self.text).unwrap();        self.text = new_text;    }     #[inline(always)]    pub fn as_slice(&self) -> ChunkSlice<'_> {        ChunkSlice {            chars: self.chars,            chars_utf16: self.chars_utf16,            newlines: self.newlines,            tabs: self.tabs,            text: &self.text,        }    }     #[inline(always)]    pub fn slice(&self, range: Range<usize>) -> ChunkSlice<'_> {        self.as_slice().slice(range)    }     #[inline(always)]    pub fn chars(&self) -> Bitmap {        self.chars    }     #[inline(always)]    pub fn tabs(&self) -> Bitmap {        self.tabs    }     #[inline(always)]    pub fn newlines(&self) -> Bitmap {        self.newlines    }     #[inline(always)]    pub fn is_char_boundary(&self, offset: usize) -> bool {        (1 as Bitmap).unbounded_shl(offset as u32) & self.chars != 0 || offset == self.text.len()    }     pub fn floor_char_boundary(&self, index: usize) -> usize {        if index >= self.text.len() {            self.text.len()        } else {            let mut i = index;            while i > 0 {                if util::is_utf8_char_boundary(self.text.as_bytes()[i]) {                    break;                }                i -= 1;            }             i        }    }     #[track_caller]    #[inline(always)]    pub fn assert_char_boundary<const PANIC: bool>(&self, offset: usize) -> bool {        if self.is_char_boundary(offset) {            return true;        }        if PANIC || cfg!(debug_assertions) {            panic_char_boundary(&self.text, offset);        } else {            log_err_char_boundary(&self.text, offset);            false        }    }} #[derive(Clone, Copy, Debug)]pub struct ChunkSlice<'a> {    chars: Bitmap,    chars_utf16: Bitmap,    newlines: Bitmap,    tabs: Bitmap,    text: &'a str,} impl Into<Chunk> for ChunkSlice<'_> {    fn into(self) -> Chunk {        Chunk {            chars: self.chars,            chars_utf16: self.chars_utf16,            newlines: self.newlines,            tabs: self.tabs,            text: self.text.try_into().unwrap(),        }    }} impl<'a> ChunkSlice<'a> {    #[inline(always)]    pub fn is_empty(&self) -> bool {        self.text.is_empty()    }     #[inline(always)]    pub fn is_char_boundary(&self, offset: usize) -> bool {        (1 as Bitmap).unbounded_shl(offset as u32) & self.chars != 0 || offset == self.text.len()    }     #[inline(always)]    pub fn split_at(self, mid: usize) -> (ChunkSlice<'a>, ChunkSlice<'a>) {        if mid == MAX_BASE {            let left = self;            let right = ChunkSlice {                chars: 0,                chars_utf16: 0,                newlines: 0,                tabs: 0,                text: "",            };            (left, right)        } else {            let mask = ((1 as Bitmap) << mid) - 1;            let (left_text, right_text) = self.text.split_at(mid);            let left = ChunkSlice {                chars: self.chars & mask,                chars_utf16: self.chars_utf16 & mask,                newlines: self.newlines & mask,                tabs: self.tabs & mask,                text: left_text,            };            let right = ChunkSlice {                chars: self.chars >> mid,                chars_utf16: self.chars_utf16 >> mid,                newlines: self.newlines >> mid,                tabs: self.tabs >> mid,                text: right_text,            };            (left, right)        }    }     #[inline(always)]    pub fn slice(self, mut range: Range<usize>) -> Self {        if range.start == MAX_BASE {            Self {                chars: 0,                chars_utf16: 0,                newlines: 0,                tabs: 0,                text: "",            }        } else {            if !self.assert_char_boundary::<false>(range.start) {                range.start = self.text.ceil_char_boundary(range.start);            }            if !self.assert_char_boundary::<false>(range.end) {                range.end = if range.end < range.start {                    range.start                } else {                    self.text.floor_char_boundary(range.end)                };            }            let mask = (1 as Bitmap)                .unbounded_shl(range.end as u32)                .wrapping_sub(1);            Self {                chars: (self.chars & mask) >> range.start,                chars_utf16: (self.chars_utf16 & mask) >> range.start,                newlines: (self.newlines & mask) >> range.start,                tabs: (self.tabs & mask) >> range.start,                text: &self.text[range],            }        }    }     #[inline(always)]    pub fn text_summary(&self) -> TextSummary {        let mut chars = 0;        let (longest_row, longest_row_chars) = self.longest_row(&mut chars);        TextSummary {            len: self.len(),            chars,            len_utf16: self.len_utf16(),            lines: self.lines(),            first_line_chars: self.first_line_chars(),            last_line_chars: self.last_line_chars(),            last_line_len_utf16: self.last_line_len_utf16(),            longest_row,            longest_row_chars,        }    }     /// Get length in bytes    #[inline(always)]    pub fn len(&self) -> usize {        self.text.len()    }     /// Get length in UTF-16 code units    #[inline(always)]    pub fn len_utf16(&self) -> OffsetUtf16 {        OffsetUtf16(self.chars_utf16.count_ones() as usize)    }     /// Get point representing number of lines and length of last line    #[inline(always)]    pub fn lines(&self) -> Point {        let row = self.newlines.count_ones();        let column = self.newlines.leading_zeros() - (Bitmap::BITS - self.text.len() as u32);        Point::new(row, column)    }     /// Get number of chars in first line    #[inline(always)]    pub fn first_line_chars(&self) -> u32 {        (self.chars & saturating_shl_mask(self.newlines.trailing_zeros())).count_ones()    }     /// Get number of chars in last line    #[inline(always)]    pub fn last_line_chars(&self) -> u32 {        (self.chars & saturating_shr_mask(self.newlines.leading_zeros())).count_ones()    }     /// Get number of UTF-16 code units in last line    #[inline(always)]    pub fn last_line_len_utf16(&self) -> u32 {        (self.chars_utf16 & saturating_shr_mask(self.newlines.leading_zeros())).count_ones()    }     /// Get the longest row in the chunk and its length in characters.    /// Calculate the total number of characters in the chunk along the way.    #[inline(always)]    pub fn longest_row(&self, total_chars: &mut usize) -> (u32, u32) {        let mut chars = self.chars;        let mut newlines = self.newlines;        *total_chars = 0;        let mut row = 0;        let mut longest_row = 0;        let mut longest_row_chars = 0;        while newlines > 0 {            let newline_ix = newlines.trailing_zeros();            let row_chars = (chars & ((1 << newline_ix) - 1)).count_ones() as u8;            *total_chars += usize::from(row_chars);            if row_chars > longest_row_chars {                longest_row = row;                longest_row_chars = row_chars;            }             newlines >>= newline_ix;            newlines >>= 1;            chars >>= newline_ix;            chars >>= 1;            row += 1;            *total_chars += 1;        }         let row_chars = chars.count_ones() as u8;        *total_chars += usize::from(row_chars);        if row_chars > longest_row_chars {            (row, row_chars as u32)        } else {            (longest_row, longest_row_chars as u32)        }    }     #[inline(always)]    pub fn offset_to_point(&self, offset: usize) -> Point {        let mask = (1 as Bitmap).unbounded_shl(offset as u32).wrapping_sub(1);        let row = (self.newlines & mask).count_ones();        let newline_ix = Bitmap::BITS - (self.newlines & mask).leading_zeros();        let column = (offset - newline_ix as usize) as u32;        Point::new(row, column)    }     #[inline(always)]    pub fn point_to_offset(&self, point: Point) -> usize {        if point.row > self.lines().row {            debug_panic!(                "point {:?} extends beyond rows for string {:?}",                point,                self.text            );            return self.len();        }         let row_offset_range = self.offset_range_for_row(point.row);        if point.column > row_offset_range.len() as u32 {            debug_panic!(                "point {:?} extends beyond row for string {:?}",                point,                self.text            );            row_offset_range.end        } else {            row_offset_range.start + point.column as usize        }    }     #[track_caller]    #[inline(always)]    pub fn assert_char_boundary<const PANIC: bool>(&self, offset: usize) -> bool {        if self.is_char_boundary(offset) {            return true;        }        if PANIC {            panic_char_boundary(self.text, offset);        } else {            log_err_char_boundary(self.text, offset);            false        }    }     pub fn floor_char_boundary(&self, index: usize) -> usize {        self.text.floor_char_boundary(index)    }     #[inline(always)]    pub fn point_to_offset_utf16(&self, point: Point) -> OffsetUtf16 {        if point.row > self.lines().row {            debug_panic!(                "point {:?} extends beyond rows for string {:?}",                point,                self.text            );            return self.len_utf16();        }        self.offset_to_offset_utf16(self.point_to_offset(point))    }     #[inline(always)]    pub fn offset_to_offset_utf16(&self, offset: usize) -> OffsetUtf16 {        let mask = (1 as Bitmap).unbounded_shl(offset as u32).wrapping_sub(1);        OffsetUtf16((self.chars_utf16 & mask).count_ones() as usize)    }     #[inline(always)]    pub fn offset_utf16_to_offset(&self, target: OffsetUtf16) -> usize {        if target.0 == 0 {            0        } else {            #[cfg(not(test))]            let chars_utf16 = self.chars_utf16;            #[cfg(test)]            let chars_utf16 = self.chars_utf16 as u128;            let ix = nth_set_bit(chars_utf16, target.0) + 1;            if ix == MAX_BASE {                MAX_BASE            } else {                let utf8_additional_len = cmp::min(                    (self.chars_utf16 >> ix).trailing_zeros() as usize,                    self.text.len() - ix,                );                ix + utf8_additional_len            }        }    }     #[inline(always)]    pub fn offset_to_point_utf16(&self, offset: usize) -> PointUtf16 {

Lines 1–500 of 1249. Load more