wezterm-font: add find-fall-back-for-codepoints concept

This commit makes some adjustments to FontConfiguration and LoadedFont
such that it the shaper is unable to resolve a (non-last-resort) font
for a set of codepoints, the locator can be used to try to find a
font that has coverage for those codepoints.

At the moment this is a bit limited:

* Only the font-config locator implements this function
* The directory based locator isn't actually an implementor of the
  locator trait and doesn't have a way to be invoked for this.

wezterm-font/src/fcwrap.rs

@@ -139,6 +139,40 @@ impl FcResultWrap {
     }
 }
 
+pub struct CharSet {
+    cset: *mut FcCharSet,
+}
+
+impl Drop for CharSet {
+    fn drop(&mut self) {
+        unsafe {
+            FcCharSetDestroy(self.cset);
+        }
+        release_object();
+    }
+}
+
+impl CharSet {
+    pub fn new() -> anyhow::Result<Self> {
+        unsafe {
+            let cset = FcCharSetCreate();
+            ensure!(!cset.is_null(), "FcCharSetCreate failed");
+            add_object();
+            Ok(Self { cset })
+        }
+    }
+
+    pub fn add(&mut self, c: char) -> anyhow::Result<()> {
+        unsafe {
+            ensure!(
+                FcCharSetAddChar(self.cset, c as u32) != 0,
+                "FcCharSetAddChar failed"
+            );
+            Ok(())
+        }
+    }
+}
+
 pub struct Pattern {
     pat: *mut FcPattern,
 }
@@ -153,6 +187,26 @@ impl Pattern {
         }
     }
 
+    pub fn add_charset(&mut self, charset: &CharSet) -> anyhow::Result<()> {
+        unsafe {
+            ensure!(
+                FcPatternAddCharSet(self.pat, b"charset\0".as_ptr() as *const i8, charset.cset)
+                    != 0,
+                "failed to add charset property"
+            );
+            Ok(())
+        }
+    }
+
+    pub fn charset_intersect_count(&self, charset: &CharSet) -> anyhow::Result<u32> {
+        unsafe {
+            let mut c = ptr::null_mut();
+            FcPatternGetCharSet(self.pat, b"charset\0".as_ptr() as *const i8, 0, &mut c);
+            ensure!(!c.is_null(), "pattern has no charset");
+            Ok(FcCharSetIntersectCount(c, charset.cset))
+        }
+    }
+
     pub fn add_string(&mut self, key: &str, value: &str) -> Result<(), Error> {
         let key = CString::new(key)?;
         let value = CString::new(value)?;
@@ -241,6 +295,29 @@ impl Pattern {
         }
     }
 
+    pub fn list(&self) -> anyhow::Result<FontSet> {
+        log::trace!("listing: {:?}", self);
+        unsafe {
+            // This defines the fields that are retrieved
+            let oset = FcObjectSetCreate();
+            ensure!(!oset.is_null(), "FcObjectSetCreate failed");
+            FcObjectSetAdd(oset, b"family\0".as_ptr() as *const i8);
+            FcObjectSetAdd(oset, b"file\0".as_ptr() as *const i8);
+            FcObjectSetAdd(oset, b"index\0".as_ptr() as *const i8);
+            FcObjectSetAdd(oset, b"charset\0".as_ptr() as *const i8);
+
+            let fonts = FcFontList(ptr::null_mut(), self.pat, oset);
+            let result = if !fonts.is_null() {
+                add_object();
+                Ok(FontSet { fonts })
+            } else {
+                Err(anyhow!("FcFontList failed"))
+            };
+            FcObjectSetDestroy(oset);
+            result
+        }
+    }
+
     pub fn sort(&self, trim: bool) -> Result<FontSet, Error> {
         unsafe {
             let mut res = FcResultWrap(0);
@@ -335,7 +412,7 @@ impl fmt::Debug for Pattern {
         // unsafe{FcPatternPrint(self.pat);}
         fmt.write_str(
             &self
-                .format("Pattern(%{+family,style,weight,slant,spacing,file,index,fontformat{%{=unparse}}})")
+                .format("Pattern(%{+family,style,weight,slant,spacing,file,index,charset,fontformat{%{=unparse}}})")
                 .unwrap(),
         )
     }

wezterm-font/src/lib.rs

@@ -5,7 +5,7 @@ use anyhow::{anyhow, Error};
 use config::{configuration, ConfigHandle, FontRasterizerSelection, TextStyle};
 use std::cell::RefCell;
 use std::collections::{HashMap, HashSet};
-use std::rc::Rc;
+use std::rc::{Rc, Weak};
 use wezterm_term::CellAttributes;
 
 mod hbwrap;
@@ -25,11 +25,12 @@ pub use crate::shaper::{FallbackIdx, FontMetrics, GlyphInfo};
 
 pub struct LoadedFont {
     rasterizers: Vec<RefCell<Option<Box<dyn FontRasterizer>>>>,
-    handles: Vec<FontDataHandle>,
-    shaper: Box<dyn FontShaper>,
+    handles: RefCell<Vec<FontDataHandle>>,
+    shaper: RefCell<Box<dyn FontShaper>>,
     metrics: FontMetrics,
     font_size: f64,
     dpi: u32,
+    font_config: Weak<FontConfigInner>,
 }
 
 impl LoadedFont {
@@ -38,11 +39,69 @@ impl LoadedFont {
     }
 
     pub fn shape(&self, text: &str) -> anyhow::Result<Vec<GlyphInfo>> {
-        self.shaper.shape(text, self.font_size, self.dpi)
+        let mut no_glyphs = vec![];
+        let result = self
+            .shaper
+            .borrow()
+            .shape(text, self.font_size, self.dpi, &mut no_glyphs);
+
+        if !no_glyphs.is_empty() {
+            no_glyphs.sort();
+            no_glyphs.dedup();
+            if let Some(font_config) = self.font_config.upgrade() {
+                match font_config
+                    .locator
+                    .locate_fallback_for_codepoints(&no_glyphs)
+                {
+                    Err(err) => log::error!(
+                        "Error: {} while resolving a fallback font for {:x?}",
+                        err,
+                        no_glyphs.iter().collect::<String>().escape_debug()
+                    ),
+                    Ok(handles) if handles.is_empty() => {
+                        log::error!(
+                            "No fonts have glyphs for {}",
+                            no_glyphs.iter().collect::<String>().escape_debug()
+                        )
+                    }
+                    Ok(extra_handles) => {
+                        let mut loaded = false;
+                        {
+                            let mut handles = self.handles.borrow_mut();
+                            for h in extra_handles {
+                                if !handles.iter().any(|existing| *existing == h) {
+                                    if crate::parser::ParsedFont::from_locator(&h).is_ok() {
+                                        let idx = handles.len() - 1;
+                                        handles.insert(idx, h);
+                                        loaded = true;
+                                    }
+                                }
+                            }
+                        }
+                        if loaded {
+                            *self.shaper.borrow_mut() = new_shaper(
+                                FontShaperSelection::get_default(),
+                                &self.handles.borrow(),
+                            )?;
+                            log::trace!("handles is now: {:#?}", self.handles);
+                            return self.shape(text);
+                        } else {
+                            log::error!(
+                                "No fonts have glyphs for {}",
+                                no_glyphs.iter().collect::<String>().escape_debug()
+                            )
+                        }
+                    }
+                }
+            }
+        }
+
+        result
     }
 
     pub fn metrics_for_idx(&self, font_idx: usize) -> anyhow::Result<FontMetrics> {
         self.shaper
+            .borrow()
             .metrics_for_idx(font_idx, self.font_size, self.dpi)
     }
 
@@ -59,7 +118,7 @@ impl LoadedFont {
         if opt_raster.is_none() {
             let raster = new_rasterizer(
                 FontRasterizerSelection::get_default(),
-                &self.handles[fallback],
+                &(self.handles.borrow())[fallback],
             )?;
             opt_raster.replace(raster);
         }
@@ -71,8 +130,7 @@ impl LoadedFont {
     }
 }
 
-/// Matches and loads fonts for a given input style
-pub struct FontConfiguration {
+struct FontConfigInner {
     fonts: RefCell<HashMap<TextStyle, Rc<LoadedFont>>>,
     metrics: RefCell<Option<FontMetrics>>,
     dpi_scale: RefCell<f64>,
@@ -81,7 +139,12 @@ pub struct FontConfiguration {
     locator: Box<dyn FontLocator>,
 }
 
-impl FontConfiguration {
+/// Matches and loads fonts for a given input style
+pub struct FontConfiguration {
+    inner: Rc<FontConfigInner>,
+}
+
+impl FontConfigInner {
     /// Create a new empty configuration
     pub fn new() -> Self {
         let locator = new_locator(FontLocatorSelection::get_default());
@@ -97,7 +160,7 @@ impl FontConfiguration {
 
     /// Given a text style, load (with caching) the font that best
     /// matches according to the fontconfig pattern.
-    pub fn resolve_font(&self, style: &TextStyle) -> anyhow::Result<Rc<LoadedFont>> {
+    fn resolve_font(&self, myself: &Rc<Self>, style: &TextStyle) -> anyhow::Result<Rc<LoadedFont>> {
         let mut fonts = self.fonts.borrow_mut();
 
         let config = configuration();
@@ -206,11 +269,12 @@ impl FontConfiguration {
 
         let loaded = Rc::new(LoadedFont {
             rasterizers,
-            handles,
-            shaper,
+            handles: RefCell::new(handles),
+            shaper: RefCell::new(shaper),
             metrics,
             font_size,
             dpi,
+            font_config: Rc::downgrade(myself),
         });
 
         fonts.insert(style.clone(), Rc::clone(&loaded));
@@ -226,15 +290,15 @@ impl FontConfiguration {
     }
 
     /// Returns the baseline font specified in the configuration
-    pub fn default_font(&self) -> anyhow::Result<Rc<LoadedFont>> {
-        self.resolve_font(&configuration().font)
+    pub fn default_font(&self, myself: &Rc<Self>) -> anyhow::Result<Rc<LoadedFont>> {
+        self.resolve_font(myself, &configuration().font)
     }
 
     pub fn get_font_scale(&self) -> f64 {
         *self.font_scale.borrow()
     }
 
-    pub fn default_font_metrics(&self) -> Result<FontMetrics, Error> {
+    pub fn default_font_metrics(&self, myself: &Rc<Self>) -> Result<FontMetrics, Error> {
         {
             let metrics = self.metrics.borrow();
             if let Some(metrics) = metrics.as_ref() {
@@ -242,7 +306,7 @@ impl FontConfiguration {
             }
         }
 
-        let font = self.default_font()?;
+        let font = self.default_font(myself)?;
         let metrics = font.metrics();
 
         *self.metrics.borrow_mut() = Some(metrics);
@@ -290,3 +354,45 @@ impl FontConfiguration {
         &config.font
     }
 }
+
+impl FontConfiguration {
+    /// Create a new empty configuration
+    pub fn new() -> Self {
+        let inner = Rc::new(FontConfigInner::new());
+        Self { inner }
+    }
+
+    /// Given a text style, load (with caching) the font that best
+    /// matches according to the fontconfig pattern.
+    pub fn resolve_font(&self, style: &TextStyle) -> anyhow::Result<Rc<LoadedFont>> {
+        self.inner.resolve_font(&self.inner, style)
+    }
+
+    pub fn change_scaling(&self, font_scale: f64, dpi_scale: f64) {
+        self.inner.change_scaling(font_scale, dpi_scale)
+    }
+
+    /// Returns the baseline font specified in the configuration
+    pub fn default_font(&self) -> anyhow::Result<Rc<LoadedFont>> {
+        self.inner.default_font(&self.inner)
+    }
+
+    pub fn get_font_scale(&self) -> f64 {
+        self.inner.get_font_scale()
+    }
+
+    pub fn default_font_metrics(&self) -> Result<FontMetrics, Error> {
+        self.inner.default_font_metrics(&self.inner)
+    }
+
+    /// Apply the defined font_rules from the user configuration to
+    /// produce the text style that best matches the supplied input
+    /// cell attributes.
+    pub fn match_style<'a>(
+        &self,
+        config: &'a ConfigHandle,
+        attrs: &CellAttributes,
+    ) -> &'a TextStyle {
+        self.inner.match_style(config, attrs)
+    }
+}

wezterm-font/src/locator/core_text.rs

@@ -90,4 +90,11 @@ impl FontLocator for CoreTextFontLocator {
 
         Ok(fonts)
     }
+
+    fn locate_fallback_for_codepoints(
+        &self,
+        _codepoints: &[char],
+    ) -> anyhow::Result<Vec<FontDataHandle>> {
+        Ok(vec![])
+    }
 }

wezterm-font/src/locator/font_config.rs

@@ -1,7 +1,8 @@
 use crate::fcwrap;
 use crate::locator::{FontDataHandle, FontLocator};
+use anyhow::Context;
 use config::FontAttributes;
-use fcwrap::Pattern as FontPattern;
+use fcwrap::{CharSet, Pattern as FontPattern};
 use std::collections::HashSet;
 use std::convert::TryInto;
 
@@ -22,6 +23,7 @@ impl FontLocator for FontConfigFontLocator {
             pattern.family(&attr.family)?;
             pattern.add_integer("weight", if attr.bold { 200 } else { 80 })?;
             pattern.add_integer("slant", if attr.italic { 100 } else { 0 })?;
+            pattern.add_string("fontformat", "TrueType")?;
             /*
             pattern.add_double("size", config.font_size * font_scale)?;
             pattern.add_double("dpi", config.dpi)?;
@@ -56,4 +58,54 @@ impl FontLocator for FontConfigFontLocator {
 
         Ok(fonts)
     }
+
+    fn locate_fallback_for_codepoints(
+        &self,
+        codepoints: &[char],
+    ) -> anyhow::Result<Vec<FontDataHandle>> {
+        let mut charset = CharSet::new()?;
+        for &c in codepoints {
+            charset.add(c)?;
+        }
+
+        let mut pattern = FontPattern::new()?;
+        pattern.add_charset(&charset)?;
+        pattern.add_string("fontformat", "TrueType")?;
+        pattern.add_integer("weight", 80)?;
+        pattern.add_integer("slant", 0)?;
+
+        let any_spacing = pattern
+            .list()
+            .context("pattern.list with no spacing constraint")?;
+        pattern.monospace()?;
+        let mono_spacing = pattern
+            .list()
+            .context("pattern.list with monospace constraint")?;
+
+        let mut fonts = vec![];
+
+        for list in &[mono_spacing, any_spacing] {
+            for pat in list.iter() {
+                let num = pat.charset_intersect_count(&charset)?;
+                if num == 0 {
+                    log::error!(
+                        "Skipping bogus font-config result {:?} because it doesn't overlap",
+                        pat
+                    );
+                    continue;
+                }
+
+                let file = pat.get_file().context("pat.get_file")?;
+
+                let handle = FontDataHandle::OnDisk {
+                    path: file.into(),
+                    index: pat.get_integer("index")?.try_into()?,
+                };
+
+                fonts.push(handle);
+            }
+        }
+
+        Ok(fonts)
+    }
 }

wezterm-font/src/locator/gdi.rs

@@ -101,4 +101,11 @@ impl FontLocator for GdiFontLocator {
 
         Ok(fonts)
     }
+
+    fn locate_fallback_for_codepoints(
+        &self,
+        _codepoints: &[char],
+    ) -> anyhow::Result<Vec<FontDataHandle>> {
+        Ok(vec![])
+    }
 }

wezterm-font/src/locator/mod.rs

@@ -13,7 +13,7 @@ pub mod gdi;
 /// The `index` parameter is the index into a font
 /// collection if the data represents a collection of
 /// fonts.
-#[derive(Clone)]
+#[derive(Clone, PartialEq, Eq)]
 pub enum FontDataHandle {
     OnDisk {
         path: PathBuf,
@@ -53,6 +53,11 @@ pub trait FontLocator {
         fonts_selection: &[FontAttributes],
         loaded: &mut HashSet<FontAttributes>,
     ) -> anyhow::Result<Vec<FontDataHandle>>;
+
+    fn locate_fallback_for_codepoints(
+        &self,
+        codepoints: &[char],
+    ) -> anyhow::Result<Vec<FontDataHandle>>;
 }
 
 pub fn new_locator(locator: FontLocatorSelection) -> Box<dyn FontLocator> {
@@ -91,4 +96,11 @@ impl FontLocator for NopSystemSource {
     ) -> anyhow::Result<Vec<FontDataHandle>> {
         Ok(vec![])
     }
+
+    fn locate_fallback_for_codepoints(
+        &self,
+        _codepoints: &[char],
+    ) -> anyhow::Result<Vec<FontDataHandle>> {
+        Ok(vec![])
+    }
 }

wezterm-font/src/shaper/allsorts.rs

@@ -40,6 +40,7 @@ impl AllsortsShaper {
         font_size: f64,
         dpi: u32,
         results: &mut Vec<GlyphInfo>,
+        no_glyphs: &mut Vec<char>,
     ) -> anyhow::Result<()> {
         let font = match self.fonts.get(font_index) {
             Some(Some(font)) => font,
@@ -53,13 +54,17 @@ impl AllsortsShaper {
                     font_size,
                     dpi,
                     results,
+                    no_glyphs,
                 );
             }
             None => {
+                // Note: since we added a last resort font, this case shouldn't
+                // ever get hit in practice.
                 // We ran out of fallback fonts, so use a replacement
-                // character that is likely to be in one of those fonts
+                // character that is likely to be in one of those fonts.
                 let mut alt_text = String::new();
-                for _c in s.chars() {
+                for c in s.chars() {
+                    no_glyphs.push(c);
                     alt_text.push('?');
                 }
                 if alt_text == s {
@@ -75,9 +80,19 @@ impl AllsortsShaper {
                     font_size,
                     dpi,
                     results,
+                    no_glyphs,
                 );
             }
         };
+
+        if font_index + 1 == self.fonts.len() {
+            // We are the last resort font, so each codepoint is considered
+            // to be worthy of a fallback lookup
+            for c in s.chars() {
+                no_glyphs.push(c);
+            }
+        }
+
         let first_pass =
             font.shape_text(s, slice_index, font_index, script, lang, font_size, dpi)?;
 
@@ -100,6 +115,7 @@ impl AllsortsShaper {
                         font_size,
                         dpi,
                         results,
+                        no_glyphs,
                     )?;
                 }
             }
@@ -110,11 +126,17 @@ impl AllsortsShaper {
 }
 
 impl FontShaper for AllsortsShaper {
-    fn shape(&self, text: &str, size: f64, dpi: u32) -> anyhow::Result<Vec<GlyphInfo>> {
+    fn shape(
+        &self,
+        text: &str,
+        size: f64,
+        dpi: u32,
+        no_glyphs: &mut Vec<char>,
+    ) -> anyhow::Result<Vec<GlyphInfo>> {
         let mut results = vec![];
         let script = allsorts::tag::LATN;
         let lang = allsorts::tag::DFLT;
-        self.shape_into(0, text, 0, script, lang, size, dpi, &mut results)?;
+        self.shape_into(0, text, 0, script, lang, size, dpi, &mut results, no_glyphs)?;
         // log::error!("shape {} into {:?}", text, results);
         Ok(results)
     }

wezterm-font/src/shaper/harfbuzz.rs

@@ -145,6 +145,7 @@ impl HarfbuzzShaper {
         s: &str,
         font_size: f64,
         dpi: u32,
+        no_glyphs: &mut Vec<char>,
     ) -> anyhow::Result<Vec<GlyphInfo>> {
         let config = configuration();
         let features: Vec<harfbuzz::hb_feature_t> = config
@@ -179,6 +180,11 @@ impl HarfbuzzShaper {
                     pair.font.shape(&mut buf, Some(features.as_slice()));
                 }
                 None => {
+                    // Note: since we added a last resort font, this case
+                    // shouldn't ever get hit in practice
+                    for c in s.chars() {
+                        no_glyphs.push(c);
+                    }
                     return Err(NoMoreFallbacksError {
                         text: s.to_string(),
                     }
@@ -187,6 +193,14 @@ impl HarfbuzzShaper {
             }
         }
 
+        if font_idx + 1 == self.fonts.len() {
+            // We are the last resort font, so each codepoint is considered
+            // to be worthy of a fallback lookup
+            for c in s.chars() {
+                no_glyphs.push(c);
+            }
+        }
+
         let hb_infos = buf.glyph_infos();
         let positions = buf.glyph_positions();
 
@@ -259,11 +273,12 @@ impl HarfbuzzShaper {
                 }
                 */
 
-                let mut shape = match self.do_shape(font_idx + 1, substr, font_size, dpi) {
+                let mut shape = match self.do_shape(font_idx + 1, substr, font_size, dpi, no_glyphs)
+                {
                     Ok(shape) => Ok(shape),
                     Err(e) => {
                         error!("{:?} for {:?}", e, substr);
-                        self.do_shape(0, &make_question_string(substr), font_size, dpi)
+                        self.do_shape(0, &make_question_string(substr), font_size, dpi, no_glyphs)
                     }
                 }?;
 
@@ -317,9 +332,15 @@ impl HarfbuzzShaper {
 }
 
 impl FontShaper for HarfbuzzShaper {
-    fn shape(&self, text: &str, size: f64, dpi: u32) -> anyhow::Result<Vec<GlyphInfo>> {
+    fn shape(
+        &self,
+        text: &str,
+        size: f64,
+        dpi: u32,
+        no_glyphs: &mut Vec<char>,
+    ) -> anyhow::Result<Vec<GlyphInfo>> {
         let start = std::time::Instant::now();
-        let result = self.do_shape(0, text, size, dpi);
+        let result = self.do_shape(0, text, size, dpi, no_glyphs);
         metrics::value!("shape.harfbuzz", start.elapsed());
         /*
         if let Ok(glyphs) = &result {

wezterm-font/src/shaper/mod.rs

@@ -54,7 +54,13 @@ pub struct FontMetrics {
 
 pub trait FontShaper {
     /// Shape text and return a vector of GlyphInfo
-    fn shape(&self, text: &str, size: f64, dpi: u32) -> anyhow::Result<Vec<GlyphInfo>>;
+    fn shape(
+        &self,
+        text: &str,
+        size: f64,
+        dpi: u32,
+        no_glyphs: &mut Vec<char>,
+    ) -> anyhow::Result<Vec<GlyphInfo>>;
 
     /// Compute the font metrics for the preferred font
     /// at the specified size.