termwiz: refactor: split line into sub-modules

termwiz/src/surface/line/cellref.rs

@@ -0,0 +1,68 @@
+use crate::cell::{Cell, CellAttributes};
+use crate::emoji::Presentation;
+
+#[derive(Debug)]
+pub enum CellRef<'a> {
+    CellRef {
+        cell_index: usize,
+        cell: &'a Cell,
+    },
+    ClusterRef {
+        cell_index: usize,
+        text: &'a str,
+        width: usize,
+        attrs: &'a CellAttributes,
+    },
+}
+
+impl<'a> CellRef<'a> {
+    pub fn cell_index(&self) -> usize {
+        match self {
+            Self::ClusterRef { cell_index, .. } | Self::CellRef { cell_index, .. } => *cell_index,
+        }
+    }
+
+    pub fn str(&self) -> &str {
+        match self {
+            Self::CellRef { cell, .. } => cell.str(),
+            Self::ClusterRef { text, .. } => text,
+        }
+    }
+
+    pub fn width(&self) -> usize {
+        match self {
+            Self::CellRef { cell, .. } => cell.width(),
+            Self::ClusterRef { width, .. } => *width,
+        }
+    }
+
+    pub fn attrs(&self) -> &CellAttributes {
+        match self {
+            Self::CellRef { cell, .. } => cell.attrs(),
+            Self::ClusterRef { attrs, .. } => attrs,
+        }
+    }
+
+    pub fn presentation(&self) -> Presentation {
+        match self {
+            Self::CellRef { cell, .. } => cell.presentation(),
+            Self::ClusterRef { text, .. } => match Presentation::for_grapheme(text) {
+                (_, Some(variation)) => variation,
+                (presentation, None) => presentation,
+            },
+        }
+    }
+
+    pub fn as_cell(&self) -> Cell {
+        match self {
+            Self::CellRef { cell, .. } => (*cell).clone(),
+            Self::ClusterRef {
+                text, width, attrs, ..
+            } => Cell::new_grapheme_with_width(text, *width, (*attrs).clone()),
+        }
+    }
+
+    pub fn same_contents(&self, other: &Self) -> bool {
+        self.str() == other.str() && self.width() == other.width() && self.attrs() == other.attrs()
+    }
+}

termwiz/src/surface/line/clusterline.rs

@@ -0,0 +1,343 @@
+use crate::cell::{Cell, CellAttributes};
+use crate::surface::line::CellRef;
+use fixedbitset::FixedBitSet;
+#[cfg(feature = "use_serde")]
+use serde::{Deserialize, Deserializer, Serialize, Serializer};
+use std::num::NonZeroU8;
+use unicode_segmentation::UnicodeSegmentation;
+
+#[cfg_attr(feature = "use_serde", derive(Serialize, Deserialize))]
+#[derive(Debug, Clone, PartialEq)]
+struct Cluster {
+    cell_width: usize,
+    attrs: CellAttributes,
+}
+
+/// Stores line data as a contiguous string and a series of
+/// clusters of attribute data describing attributed ranges
+/// within the line
+#[cfg_attr(feature = "use_serde", derive(Serialize, Deserialize))]
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) struct ClusteredLine {
+    pub text: String,
+    #[cfg_attr(
+        feature = "use_serde",
+        serde(
+            deserialize_with = "deserialize_bitset",
+            serialize_with = "serialize_bitset"
+        )
+    )]
+    is_double_wide: Option<FixedBitSet>,
+    clusters: Vec<Cluster>,
+    len: usize,
+    last_cell_width: Option<NonZeroU8>,
+}
+
+#[cfg(feature = "use_serde")]
+fn deserialize_bitset<'de, D>(deserializer: D) -> Result<Option<FixedBitSet>, D::Error>
+where
+    D: Deserializer<'de>,
+{
+    let wide_indices = <Vec<usize>>::deserialize(deserializer)?;
+    if wide_indices.is_empty() {
+        Ok(None)
+    } else {
+        let max_idx = wide_indices.iter().max().unwrap_or(&1);
+        let mut bitset = FixedBitSet::with_capacity(max_idx + 1);
+        for idx in wide_indices {
+            bitset.set(idx, true);
+        }
+        Ok(Some(bitset))
+    }
+}
+
+/// Serialize the bitset as a vector of the indices of just the 1 bits;
+/// the thesis is that most of the cells on a given line are single width.
+/// That may not be strictly true for users that heavily use asian scripts,
+/// but we'll start with this and see if we need to improve it.
+#[cfg(feature = "use_serde")]
+fn serialize_bitset<S>(value: &Option<FixedBitSet>, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    let mut wide_indices: Vec<usize> = vec![];
+    if let Some(bits) = value {
+        for idx in bits.ones() {
+            wide_indices.push(idx);
+        }
+    }
+    wide_indices.serialize(serializer)
+}
+
+impl ClusteredLine {
+    pub fn new() -> Self {
+        Self {
+            text: String::with_capacity(80),
+            is_double_wide: None,
+            clusters: vec![],
+            len: 0,
+            last_cell_width: None,
+        }
+    }
+
+    pub fn to_cell_vec(&self) -> Vec<Cell> {
+        let mut cells = vec![];
+
+        for c in self.iter() {
+            cells.push(c.as_cell());
+            for _ in 1..c.width() {
+                cells.push(Cell::blank_with_attrs(c.attrs().clone()));
+            }
+        }
+
+        cells
+    }
+
+    pub fn from_cell_vec<'a>(hint: usize, iter: impl Iterator<Item = CellRef<'a>>) -> Self {
+        let mut last_cluster: Option<Cluster> = None;
+        let mut is_double_wide = FixedBitSet::with_capacity(hint);
+        let mut text = String::new();
+        let mut clusters = vec![];
+        let mut any_double = false;
+        let mut len = 0;
+        let mut last_cell_width = None;
+
+        for cell in iter {
+            len += cell.width();
+            last_cell_width = NonZeroU8::new(1);
+
+            if cell.width() > 1 {
+                any_double = true;
+                is_double_wide.set(cell.cell_index(), true);
+            }
+
+            text.push_str(cell.str());
+
+            last_cluster = match last_cluster.take() {
+                None => Some(Cluster {
+                    cell_width: cell.width(),
+                    attrs: cell.attrs().clone(),
+                }),
+                Some(cluster) if cluster.attrs != *cell.attrs() => {
+                    clusters.push(cluster);
+                    Some(Cluster {
+                        cell_width: cell.width(),
+                        attrs: cell.attrs().clone(),
+                    })
+                }
+                Some(mut cluster) => {
+                    cluster.cell_width += cell.width();
+                    Some(cluster)
+                }
+            };
+        }
+
+        if let Some(cluster) = last_cluster.take() {
+            clusters.push(cluster);
+        }
+
+        Self {
+            text,
+            is_double_wide: if any_double {
+                Some(is_double_wide)
+            } else {
+                None
+            },
+            clusters,
+            len,
+            last_cell_width,
+        }
+    }
+
+    pub fn len(&self) -> usize {
+        self.len
+    }
+
+    fn is_double_wide(&self, cell_index: usize) -> bool {
+        match &self.is_double_wide {
+            Some(bitset) => bitset.contains(cell_index),
+            None => false,
+        }
+    }
+
+    pub fn iter(&self) -> ClusterLineCellIter {
+        let mut clusters = self.clusters.iter();
+        let cluster = clusters.next();
+        ClusterLineCellIter {
+            graphemes: self.text.graphemes(true),
+            clusters,
+            cluster,
+            idx: 0,
+            cluster_total: 0,
+            line: self,
+        }
+    }
+
+    pub fn append_grapheme(&mut self, text: &str, cell_width: usize, attrs: CellAttributes) {
+        let new_cluster = match self.clusters.last() {
+            Some(cluster) => cluster.attrs != attrs,
+            None => true,
+        };
+        let new_cell_index = self.len;
+        if new_cluster {
+            self.clusters.push(Cluster { attrs, cell_width });
+        } else if let Some(cluster) = self.clusters.last_mut() {
+            cluster.cell_width += cell_width;
+        }
+        self.text.push_str(text);
+
+        if cell_width > 1 {
+            let bitset = match self.is_double_wide.take() {
+                Some(mut bitset) => {
+                    bitset.grow(new_cell_index + 1);
+                    bitset.set(new_cell_index, true);
+                    bitset
+                }
+                None => {
+                    let mut bitset = FixedBitSet::with_capacity(new_cell_index + 1);
+                    bitset.set(new_cell_index, true);
+                    bitset
+                }
+            };
+            self.is_double_wide.replace(bitset);
+        }
+        self.last_cell_width = NonZeroU8::new(cell_width as u8);
+        self.len += cell_width;
+    }
+
+    pub fn append(&mut self, cell: Cell) {
+        let new_cluster = match self.clusters.last() {
+            Some(cluster) => cluster.attrs != *cell.attrs(),
+            None => true,
+        };
+        let new_cell_index = self.len;
+        let cell_width = cell.width();
+        if new_cluster {
+            self.clusters.push(Cluster {
+                attrs: (*cell.attrs()).clone(),
+                cell_width,
+            });
+        } else if let Some(cluster) = self.clusters.last_mut() {
+            cluster.cell_width += cell_width;
+        }
+        self.text.push_str(cell.str());
+
+        if cell_width > 1 {
+            let bitset = match self.is_double_wide.take() {
+                Some(mut bitset) => {
+                    bitset.grow(new_cell_index + 1);
+                    bitset.set(new_cell_index, true);
+                    bitset
+                }
+                None => {
+                    let mut bitset = FixedBitSet::with_capacity(new_cell_index + 1);
+                    bitset.set(new_cell_index, true);
+                    bitset
+                }
+            };
+            self.is_double_wide.replace(bitset);
+        }
+        self.last_cell_width = NonZeroU8::new(cell_width as u8);
+        self.len += cell_width;
+    }
+
+    pub fn prune_trailing_blanks(&mut self) -> bool {
+        let num_spaces = self.text.chars().rev().take_while(|&c| c == ' ').count();
+        if num_spaces == 0 {
+            return false;
+        }
+
+        let blank = CellAttributes::blank();
+        let mut pruned = false;
+        for _ in 0..num_spaces {
+            let mut need_pop = false;
+            if let Some(cluster) = self.clusters.last_mut() {
+                if cluster.attrs != blank {
+                    break;
+                }
+                cluster.cell_width -= 1;
+                self.text.pop();
+                self.len -= 1;
+                self.last_cell_width.take();
+                pruned = true;
+                if cluster.cell_width == 0 {
+                    need_pop = true;
+                }
+            }
+            if need_pop {
+                self.clusters.pop();
+            }
+        }
+
+        pruned
+    }
+
+    fn compute_last_cell_width(&mut self) -> Option<NonZeroU8> {
+        if self.last_cell_width.is_none() {
+            if let Some(last_cell) = self.iter().last() {
+                self.last_cell_width = NonZeroU8::new(last_cell.width() as u8);
+            }
+        }
+        self.last_cell_width
+    }
+
+    pub fn set_last_cell_was_wrapped(&mut self, wrapped: bool) {
+        if let Some(width) = self.compute_last_cell_width() {
+            let width = width.get() as usize;
+            if let Some(last_cluster) = self.clusters.last_mut() {
+                let mut attrs = last_cluster.attrs.clone();
+                attrs.set_wrapped(wrapped);
+
+                if last_cluster.cell_width == width {
+                    // Re-purpose final cluster
+                    last_cluster.attrs = attrs;
+                } else {
+                    last_cluster.cell_width -= width;
+                    self.clusters.push(Cluster {
+                        cell_width: width,
+                        attrs,
+                    });
+                }
+            }
+        }
+    }
+}
+
+pub(crate) struct ClusterLineCellIter<'a> {
+    graphemes: unicode_segmentation::Graphemes<'a>,
+    clusters: std::slice::Iter<'a, Cluster>,
+    cluster: Option<&'a Cluster>,
+    idx: usize,
+    cluster_total: usize,
+    line: &'a ClusteredLine,
+}
+
+impl<'a> Iterator for ClusterLineCellIter<'a> {
+    type Item = CellRef<'a>;
+
+    fn next(&mut self) -> Option<CellRef<'a>> {
+        let text = self.graphemes.next()?;
+
+        let cell_index = self.idx;
+        let width = if self.line.is_double_wide(cell_index) {
+            2
+        } else {
+            1
+        };
+        self.idx += width;
+        self.cluster_total += width;
+        let attrs = &self.cluster.as_ref()?.attrs;
+
+        if self.cluster_total >= self.cluster.as_ref()?.cell_width {
+            self.cluster = self.clusters.next();
+            self.cluster_total = 0;
+        }
+
+        Some(CellRef::ClusterRef {
+            cell_index,
+            width,
+            text,
+            attrs,
+        })
+    }
+}

termwiz/src/surface/line/linebits.rs

@@ -0,0 +1,56 @@
+use bitflags::bitflags;
+#[cfg(feature = "use_serde")]
+use serde::{Deserialize, Serialize};
+
+bitflags! {
+    #[cfg_attr(feature="use_serde", derive(Serialize, Deserialize))]
+    pub(crate) struct LineBits : u16 {
+        const NONE = 0;
+        /// The line contains 1+ cells with explicit hyperlinks set
+        const HAS_HYPERLINK = 1<<1;
+        /// true if we have scanned for implicit hyperlinks
+        const SCANNED_IMPLICIT_HYPERLINKS = 1<<2;
+        /// true if we found implicit hyperlinks in the last scan
+        const HAS_IMPLICIT_HYPERLINKS = 1<<3;
+
+        /// true if this line should be displayed with
+        /// foreground/background colors reversed
+        const REVERSE = 1<<4;
+
+        /// true if this line should be displayed with
+        /// in double-width
+        const DOUBLE_WIDTH = 1<<5;
+
+        /// true if this line should be displayed
+        /// as double-height top-half
+        const DOUBLE_HEIGHT_TOP = 1<<6;
+
+        /// true if this line should be displayed
+        /// as double-height bottom-half
+        const DOUBLE_HEIGHT_BOTTOM = 1<<7;
+
+        const DOUBLE_WIDTH_HEIGHT_MASK =
+            Self::DOUBLE_WIDTH.bits |
+            Self::DOUBLE_HEIGHT_TOP.bits |
+            Self::DOUBLE_HEIGHT_BOTTOM.bits;
+
+        /// true if the line should have the bidi algorithm
+        /// applied as part of presentation.
+        /// This corresponds to the "implicit" bidi modes
+        /// described in
+        /// <https://terminal-wg.pages.freedesktop.org/bidi/recommendation/basic-modes.html>
+        const BIDI_ENABLED = 1<<0;
+
+        /// true if the line base direction is RTL.
+        /// When BIDI_ENABLED is also true, this is passed to the bidi algorithm.
+        /// When rendering, the line will be rendered from RTL.
+        const RTL = 1<<8;
+
+        /// true if the direction for the line should be auto-detected
+        /// when BIDI_ENABLED is also true.
+        /// If false, the direction is taken from the RTL bit only.
+        /// Otherwise, the auto-detect direction is used, falling back
+        /// to the direction specified by the RTL bit.
+        const AUTO_DETECT_DIRECTION = 1<<9;
+    }
+}

termwiz/src/surface/line/mod.rs

@@ -0,0 +1,10 @@
+mod cellref;
+mod clusterline;
+mod line;
+mod linebits;
+mod storage;
+mod test;
+mod vecstorage;
+
+pub use cellref::CellRef;
+pub use line::{DoubleClickRange, Line};

termwiz/src/surface/line/storage.rs

@@ -0,0 +1,28 @@
+use crate::surface::line::cellref::CellRef;
+use crate::surface::line::clusterline::{ClusterLineCellIter, ClusteredLine};
+use crate::surface::line::vecstorage::{VecStorage, VecStorageIter};
+#[cfg(feature = "use_serde")]
+use serde::{Deserialize, Serialize};
+
+#[cfg_attr(feature = "use_serde", derive(Serialize, Deserialize))]
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) enum CellStorage {
+    V(VecStorage),
+    C(ClusteredLine),
+}
+
+pub(crate) enum VisibleCellIter<'a> {
+    V(VecStorageIter<'a>),
+    C(ClusterLineCellIter<'a>),
+}
+
+impl<'a> Iterator for VisibleCellIter<'a> {
+    type Item = CellRef<'a>;
+
+    fn next(&mut self) -> Option<CellRef<'a>> {
+        match self {
+            Self::V(iter) => iter.next(),
+            Self::C(iter) => iter.next(),
+        }
+    }
+}

termwiz/src/surface/line/test.rs

@@ -0,0 +1,585 @@
+#![cfg(test)]
+
+use super::*;
+use crate::cell::{Cell, CellAttributes};
+use crate::hyperlink::{Hyperlink, Rule};
+use crate::surface::line::clusterline::ClusteredLine;
+use crate::surface::SEQ_ZERO;
+use k9::assert_equal as assert_eq;
+use std::sync::Arc;
+
+#[test]
+fn hyperlinks() {
+    let text = "❤ 😍🤢 http://example.com \u{1f468}\u{1f3fe}\u{200d}\u{1f9b0} http://example.com";
+
+    let rules = vec![
+        Rule::new(r"\b\w+://(?:[\w.-]+)\.[a-z]{2,15}\S*\b", "$0").unwrap(),
+        Rule::new(r"\b\w+@[\w-]+(\.[\w-]+)+\b", "mailto:$0").unwrap(),
+    ];
+
+    let hyperlink = Arc::new(Hyperlink::new_implicit("http://example.com"));
+    let hyperlink_attr = CellAttributes::default()
+        .set_hyperlink(Some(hyperlink.clone()))
+        .clone();
+
+    let mut line: Line = text.into();
+    line.scan_and_create_hyperlinks(&rules);
+    assert!(line.has_hyperlink());
+    assert_eq!(
+        line.coerce_vec_storage().to_vec(),
+        vec![
+            Cell::new_grapheme("❤", CellAttributes::default(), None),
+            Cell::new(' ', CellAttributes::default()), // double width spacer
+            Cell::new_grapheme("😍", CellAttributes::default(), None),
+            Cell::new(' ', CellAttributes::default()), // double width spacer
+            Cell::new_grapheme("🤢", CellAttributes::default(), None),
+            Cell::new(' ', CellAttributes::default()), // double width spacer
+            Cell::new(' ', CellAttributes::default()),
+            Cell::new('h', hyperlink_attr.clone()),
+            Cell::new('t', hyperlink_attr.clone()),
+            Cell::new('t', hyperlink_attr.clone()),
+            Cell::new('p', hyperlink_attr.clone()),
+            Cell::new(':', hyperlink_attr.clone()),
+            Cell::new('/', hyperlink_attr.clone()),
+            Cell::new('/', hyperlink_attr.clone()),
+            Cell::new('e', hyperlink_attr.clone()),
+            Cell::new('x', hyperlink_attr.clone()),
+            Cell::new('a', hyperlink_attr.clone()),
+            Cell::new('m', hyperlink_attr.clone()),
+            Cell::new('p', hyperlink_attr.clone()),
+            Cell::new('l', hyperlink_attr.clone()),
+            Cell::new('e', hyperlink_attr.clone()),
+            Cell::new('.', hyperlink_attr.clone()),
+            Cell::new('c', hyperlink_attr.clone()),
+            Cell::new('o', hyperlink_attr.clone()),
+            Cell::new('m', hyperlink_attr.clone()),
+            Cell::new(' ', CellAttributes::default()),
+            Cell::new_grapheme(
+                // man: dark skin tone, red hair ZWJ emoji grapheme
+                "\u{1f468}\u{1f3fe}\u{200d}\u{1f9b0}",
+                CellAttributes::default(),
+                None,
+            ),
+            Cell::new(' ', CellAttributes::default()), // double width spacer
+            Cell::new(' ', CellAttributes::default()),
+            Cell::new('h', hyperlink_attr.clone()),
+            Cell::new('t', hyperlink_attr.clone()),
+            Cell::new('t', hyperlink_attr.clone()),
+            Cell::new('p', hyperlink_attr.clone()),
+            Cell::new(':', hyperlink_attr.clone()),
+            Cell::new('/', hyperlink_attr.clone()),
+            Cell::new('/', hyperlink_attr.clone()),
+            Cell::new('e', hyperlink_attr.clone()),
+            Cell::new('x', hyperlink_attr.clone()),
+            Cell::new('a', hyperlink_attr.clone()),
+            Cell::new('m', hyperlink_attr.clone()),
+            Cell::new('p', hyperlink_attr.clone()),
+            Cell::new('l', hyperlink_attr.clone()),
+            Cell::new('e', hyperlink_attr.clone()),
+            Cell::new('.', hyperlink_attr.clone()),
+            Cell::new('c', hyperlink_attr.clone()),
+            Cell::new('o', hyperlink_attr.clone()),
+            Cell::new('m', hyperlink_attr.clone()),
+        ]
+    );
+}
+
+#[test]
+fn double_click_range_bounds() {
+    let line: Line = "hello".into();
+    let r = line.compute_double_click_range(200, |_| true);
+    assert_eq!(r, DoubleClickRange::Range(200..200));
+}
+
+#[test]
+fn cluster_representation_basic() {
+    let line: Line = "hello".into();
+    let mut compressed = line.clone();
+    compressed.compress_for_scrollback();
+    k9::snapshot!(
+        &compressed.cells,
+        r#"
+C(
+    ClusteredLine {
+        text: "hello",
+        is_double_wide: None,
+        clusters: [
+            Cluster {
+                cell_width: 5,
+                attrs: CellAttributes {
+                    attributes: 0,
+                    intensity: Normal,
+                    underline: None,
+                    blink: None,
+                    italic: false,
+                    reverse: false,
+                    strikethrough: false,
+                    invisible: false,
+                    wrapped: false,
+                    overline: false,
+                    semantic_type: Output,
+                    foreground: Default,
+                    background: Default,
+                    fat: None,
+                },
+            },
+        ],
+        len: 5,
+        last_cell_width: Some(
+            1,
+        ),
+    },
+)
+"#
+    );
+    compressed.coerce_vec_storage();
+    assert_eq!(line, compressed);
+}
+
+#[test]
+fn cluster_representation_double_width() {
+    let line: Line = "❤ 😍🤢he❤ 😍🤢llo❤ 😍🤢".into();
+    let mut compressed = line.clone();
+    compressed.compress_for_scrollback();
+    k9::snapshot!(
+        &compressed.cells,
+        r#"
+C(
+    ClusteredLine {
+        text: "❤ 😍🤢he❤ 😍🤢llo❤ 😍🤢",
+        is_double_wide: Some(
+            FixedBitSet {
+                data: [
+                    2626580,
+                ],
+                length: 23,
+            },
+        ),
+        clusters: [
+            Cluster {
+                cell_width: 23,
+                attrs: CellAttributes {
+                    attributes: 0,
+                    intensity: Normal,
+                    underline: None,
+                    blink: None,
+                    italic: false,
+                    reverse: false,
+                    strikethrough: false,
+                    invisible: false,
+                    wrapped: false,
+                    overline: false,
+                    semantic_type: Output,
+                    foreground: Default,
+                    background: Default,
+                    fat: None,
+                },
+            },
+        ],
+        len: 23,
+        last_cell_width: Some(
+            1,
+        ),
+    },
+)
+"#
+    );
+    compressed.coerce_vec_storage();
+    assert_eq!(line, compressed);
+}
+
+#[test]
+fn cluster_representation_empty() {
+    let line = Line::from_cells(vec![], SEQ_ZERO);
+
+    let mut compressed = line.clone();
+    compressed.compress_for_scrollback();
+    k9::snapshot!(
+        &compressed.cells,
+        r#"
+C(
+    ClusteredLine {
+        text: "",
+        is_double_wide: None,
+        clusters: [],
+        len: 0,
+        last_cell_width: None,
+    },
+)
+"#
+    );
+    compressed.coerce_vec_storage();
+    assert_eq!(line, compressed);
+}
+
+#[test]
+fn cluster_wrap_last() {
+    let mut line: Line = "hello".into();
+    line.compress_for_scrollback();
+    line.set_last_cell_was_wrapped(true, 1);
+    k9::snapshot!(
+        line,
+        r#"
+Line {
+    cells: C(
+        ClusteredLine {
+            text: "hello",
+            is_double_wide: None,
+            clusters: [
+                Cluster {
+                    cell_width: 4,
+                    attrs: CellAttributes {
+                        attributes: 0,
+                        intensity: Normal,
+                        underline: None,
+                        blink: None,
+                        italic: false,
+                        reverse: false,
+                        strikethrough: false,
+                        invisible: false,
+                        wrapped: false,
+                        overline: false,
+                        semantic_type: Output,
+                        foreground: Default,
+                        background: Default,
+                        fat: None,
+                    },
+                },
+                Cluster {
+                    cell_width: 1,
+                    attrs: CellAttributes {
+                        attributes: 2048,
+                        intensity: Normal,
+                        underline: None,
+                        blink: None,
+                        italic: false,
+                        reverse: false,
+                        strikethrough: false,
+                        invisible: false,
+                        wrapped: true,
+                        overline: false,
+                        semantic_type: Output,
+                        foreground: Default,
+                        background: Default,
+                        fat: None,
+                    },
+                },
+            ],
+            len: 5,
+            last_cell_width: Some(
+                1,
+            ),
+        },
+    ),
+    zones: [],
+    seqno: 1,
+    bits: NONE,
+}
+"#
+    );
+}
+
+fn bold() -> CellAttributes {
+    use crate::cell::Intensity;
+    let mut attr = CellAttributes::default();
+    attr.set_intensity(Intensity::Bold);
+    attr
+}
+
+#[test]
+fn cluster_representation_attributes() {
+    let line = Line::from_cells(
+        vec![
+            Cell::new_grapheme("a", CellAttributes::default(), None),
+            Cell::new_grapheme("b", bold(), None),
+            Cell::new_grapheme("c", CellAttributes::default(), None),
+            Cell::new_grapheme("d", bold(), None),
+        ],
+        SEQ_ZERO,
+    );
+
+    let mut compressed = line.clone();
+    compressed.compress_for_scrollback();
+    k9::snapshot!(
+        &compressed.cells,
+        r#"
+C(
+    ClusteredLine {
+        text: "abcd",
+        is_double_wide: None,
+        clusters: [
+            Cluster {
+                cell_width: 1,
+                attrs: CellAttributes {
+                    attributes: 0,
+                    intensity: Normal,
+                    underline: None,
+                    blink: None,
+                    italic: false,
+                    reverse: false,
+                    strikethrough: false,
+                    invisible: false,
+                    wrapped: false,
+                    overline: false,
+                    semantic_type: Output,
+                    foreground: Default,
+                    background: Default,
+                    fat: None,
+                },
+            },
+            Cluster {
+                cell_width: 1,
+                attrs: CellAttributes {
+                    attributes: 1,
+                    intensity: Bold,
+                    underline: None,
+                    blink: None,
+                    italic: false,
+                    reverse: false,
+                    strikethrough: false,
+                    invisible: false,
+                    wrapped: false,
+                    overline: false,
+                    semantic_type: Output,
+                    foreground: Default,
+                    background: Default,
+                    fat: None,
+                },
+            },
+            Cluster {
+                cell_width: 1,
+                attrs: CellAttributes {
+                    attributes: 0,
+                    intensity: Normal,
+                    underline: None,
+                    blink: None,
+                    italic: false,
+                    reverse: false,
+                    strikethrough: false,
+                    invisible: false,
+                    wrapped: false,
+                    overline: false,
+                    semantic_type: Output,
+                    foreground: Default,
+                    background: Default,
+                    fat: None,
+                },
+            },
+            Cluster {
+                cell_width: 1,
+                attrs: CellAttributes {
+                    attributes: 1,
+                    intensity: Bold,
+                    underline: None,
+                    blink: None,
+                    italic: false,
+                    reverse: false,
+                    strikethrough: false,
+                    invisible: false,
+                    wrapped: false,
+                    overline: false,
+                    semantic_type: Output,
+                    foreground: Default,
+                    background: Default,
+                    fat: None,
+                },
+            },
+        ],
+        len: 4,
+        last_cell_width: Some(
+            1,
+        ),
+    },
+)
+"#
+    );
+    compressed.coerce_vec_storage();
+    assert_eq!(line, compressed);
+}
+
+#[test]
+fn cluster_append() {
+    let mut cl = ClusteredLine::new();
+    cl.append(Cell::new_grapheme("h", CellAttributes::default(), None));
+    cl.append(Cell::new_grapheme("e", CellAttributes::default(), None));
+    cl.append(Cell::new_grapheme("l", bold(), None));
+    cl.append(Cell::new_grapheme("l", CellAttributes::default(), None));
+    cl.append(Cell::new_grapheme("o", CellAttributes::default(), None));
+    k9::snapshot!(
+        cl,
+        r#"
+ClusteredLine {
+    text: "hello",
+    is_double_wide: None,
+    clusters: [
+        Cluster {
+            cell_width: 2,
+            attrs: CellAttributes {
+                attributes: 0,
+                intensity: Normal,
+                underline: None,
+                blink: None,
+                italic: false,
+                reverse: false,
+                strikethrough: false,
+                invisible: false,
+                wrapped: false,
+                overline: false,
+                semantic_type: Output,
+                foreground: Default,
+                background: Default,
+                fat: None,
+            },
+        },
+        Cluster {
+            cell_width: 1,
+            attrs: CellAttributes {
+                attributes: 1,
+                intensity: Bold,
+                underline: None,
+                blink: None,
+                italic: false,
+                reverse: false,
+                strikethrough: false,
+                invisible: false,
+                wrapped: false,
+                overline: false,
+                semantic_type: Output,
+                foreground: Default,
+                background: Default,
+                fat: None,
+            },
+        },
+        Cluster {
+            cell_width: 2,
+            attrs: CellAttributes {
+                attributes: 0,
+                intensity: Normal,
+                underline: None,
+                blink: None,
+                italic: false,
+                reverse: false,
+                strikethrough: false,
+                invisible: false,
+                wrapped: false,
+                overline: false,
+                semantic_type: Output,
+                foreground: Default,
+                background: Default,
+                fat: None,
+            },
+        },
+    ],
+    len: 5,
+    last_cell_width: Some(
+        1,
+    ),
+}
+"#
+    );
+}
+
+#[test]
+fn cluster_line_new() {
+    let mut line = Line::new(1);
+    line.set_cell(
+        0,
+        Cell::new_grapheme("h", CellAttributes::default(), None),
+        1,
+    );
+    line.set_cell(
+        1,
+        Cell::new_grapheme("e", CellAttributes::default(), None),
+        2,
+    );
+    line.set_cell(2, Cell::new_grapheme("l", bold(), None), 3);
+    line.set_cell(
+        3,
+        Cell::new_grapheme("l", CellAttributes::default(), None),
+        4,
+    );
+    line.set_cell(
+        4,
+        Cell::new_grapheme("o", CellAttributes::default(), None),
+        5,
+    );
+    k9::snapshot!(
+        line,
+        r#"
+Line {
+    cells: C(
+        ClusteredLine {
+            text: "hello",
+            is_double_wide: None,
+            clusters: [
+                Cluster {
+                    cell_width: 2,
+                    attrs: CellAttributes {
+                        attributes: 0,
+                        intensity: Normal,
+                        underline: None,
+                        blink: None,
+                        italic: false,
+                        reverse: false,
+                        strikethrough: false,
+                        invisible: false,
+                        wrapped: false,
+                        overline: false,
+                        semantic_type: Output,
+                        foreground: Default,
+                        background: Default,
+                        fat: None,
+                    },
+                },
+                Cluster {
+                    cell_width: 1,
+                    attrs: CellAttributes {
+                        attributes: 1,
+                        intensity: Bold,
+                        underline: None,
+                        blink: None,
+                        italic: false,
+                        reverse: false,
+                        strikethrough: false,
+                        invisible: false,
+                        wrapped: false,
+                        overline: false,
+                        semantic_type: Output,
+                        foreground: Default,
+                        background: Default,
+                        fat: None,
+                    },
+                },
+                Cluster {
+                    cell_width: 2,
+                    attrs: CellAttributes {
+                        attributes: 0,
+                        intensity: Normal,
+                        underline: None,
+                        blink: None,
+                        italic: false,
+                        reverse: false,
+                        strikethrough: false,
+                        invisible: false,
+                        wrapped: false,
+                        overline: false,
+                        semantic_type: Output,
+                        foreground: Default,
+                        background: Default,
+                        fat: None,
+                    },
+                },
+            ],
+            len: 5,
+            last_cell_width: Some(
+                1,
+            ),
+        },
+    ),
+    zones: [],
+    seqno: 5,
+    bits: NONE,
+}
+"#
+    );
+}

termwiz/src/surface/line/vecstorage.rs

@@ -0,0 +1,103 @@
+use crate::cell::Cell;
+use crate::surface::line::cellref::CellRef;
+#[cfg(feature = "use_serde")]
+use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+use unicode_segmentation::UnicodeSegmentation;
+
+#[cfg_attr(feature = "use_serde", derive(Serialize, Deserialize))]
+#[derive(Debug, Clone, PartialEq)]
+pub(crate) struct VecStorage {
+    cells: Vec<Cell>,
+}
+
+impl VecStorage {
+    pub(crate) fn new(cells: Vec<Cell>) -> Self {
+        Self { cells }
+    }
+
+    pub(crate) fn set_cell(&mut self, idx: usize, mut cell: Cell, clear_image_placement: bool) {
+        if !clear_image_placement {
+            if let Some(images) = self.cells[idx].attrs().images() {
+                for image in images {
+                    if image.has_placement_id() {
+                        cell.attrs_mut().attach_image(Box::new(image));
+                    }
+                }
+            }
+        }
+        self.cells[idx] = cell;
+    }
+
+    pub(crate) fn scan_and_create_hyperlinks(
+        &mut self,
+        line: &str,
+        matches: Vec<crate::hyperlink::RuleMatch>,
+    ) -> bool {
+        // The capture range is measured in bytes but we need to translate
+        // that to the index of the column.  This is complicated a bit further
+        // because double wide sequences have a blank column cell after them
+        // in the cells array, but the string we match against excludes that
+        // string.
+        let mut cell_idx = 0;
+        let mut has_implicit_hyperlinks = false;
+        for (byte_idx, _grapheme) in line.grapheme_indices(true) {
+            let cell = &mut self.cells[cell_idx];
+            let mut matched = false;
+            for m in &matches {
+                if m.range.contains(&byte_idx) {
+                    let attrs = cell.attrs_mut();
+                    // Don't replace existing links
+                    if attrs.hyperlink().is_none() {
+                        attrs.set_hyperlink(Some(Arc::clone(&m.link)));
+                        matched = true;
+                    }
+                }
+            }
+            cell_idx += cell.width();
+            if matched {
+                has_implicit_hyperlinks = true;
+            }
+        }
+
+        has_implicit_hyperlinks
+    }
+}
+
+impl std::ops::Deref for VecStorage {
+    type Target = Vec<Cell>;
+
+    fn deref(&self) -> &Vec<Cell> {
+        &self.cells
+    }
+}
+
+impl std::ops::DerefMut for VecStorage {
+    fn deref_mut(&mut self) -> &mut Vec<Cell> {
+        &mut self.cells
+    }
+}
+
+/// Iterates over a slice of Cell, yielding only visible cells
+pub(crate) struct VecStorageIter<'a> {
+    pub cells: std::slice::Iter<'a, Cell>,
+    pub idx: usize,
+    pub skip_width: usize,
+}
+
+impl<'a> Iterator for VecStorageIter<'a> {
+    type Item = CellRef<'a>;
+
+    fn next(&mut self) -> Option<CellRef<'a>> {
+        while self.skip_width > 0 {
+            self.skip_width -= 1;
+            let _ = self.cells.next()?;
+            self.idx += 1;
+        }
+        let cell = self.cells.next()?;
+        let cell_index = self.idx;
+        self.idx += 1;
+        self.skip_width = cell.width().saturating_sub(1);
+        Some(CellRef::CellRef { cell_index, cell })
+    }
+}