diff --git a/.gitignore b/.gitignore
index 3a69b9f9e..77c46a077 100644
--- a/.gitignore
+++ b/.gitignore
@@ -25,3 +25,4 @@
 /wezterm*-src.tar.gz
 /wezterm-linuxbrew.rb
 /wezterm.rb
+/perf.data*
diff --git a/Cargo.toml b/Cargo.toml
index f79e48929..77c5acc3a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -3,3 +3,4 @@ members = ["wezterm-mux-server", "wezterm", "wezterm-gui", "strip-ansi-escapes"]
 
 [profile.release]
 opt-level = 3
+# debug = 1
diff --git a/wezterm-font/src/ftwrap.rs b/wezterm-font/src/ftwrap.rs
index 7e0906a94..1636f2c61 100644
--- a/wezterm-font/src/ftwrap.rs
+++ b/wezterm-font/src/ftwrap.rs
@@ -63,6 +63,7 @@ pub fn compute_load_flags_from_config() -> (i32, FT_Render_Mode) {
 pub struct Face {
     pub face: FT_Face,
     _bytes: Vec<u8>,
+    size: Option<FaceSize>,
 }
 
 impl Drop for Face {
@@ -73,14 +74,27 @@ impl Drop for Face {
     }
 }
 
+struct FaceSize {
+    size: f64,
+    dpi: u32,
+    cell_width: f64,
+    cell_height: f64,
+}
+
 impl Face {
     /// This is a wrapper around set_char_size and select_size
     /// that accounts for some weirdness with eg: color emoji
-    pub fn set_font_size(&mut self, size: f64, dpi: u32) -> anyhow::Result<(f64, f64)> {
-        log::debug!("set_char_size {} dpi={}", size, dpi);
+    pub fn set_font_size(&mut self, point_size: f64, dpi: u32) -> anyhow::Result<(f64, f64)> {
+        if let Some(face_size) = self.size.as_ref() {
+            if face_size.size == point_size && face_size.dpi == dpi {
+                return Ok((face_size.cell_width, face_size.cell_height));
+            }
+        }
+
+        log::debug!("set_char_size computing {} dpi={}", point_size, dpi);
         // Scaling before truncating to integer minimizes the chances of hitting
         // the fallback code for set_pixel_sizes below.
-        let size = (size * 64.0) as FT_F26Dot6;
+        let size = (point_size * 64.0) as FT_F26Dot6;
 
         let (cell_width, cell_height) = match self.set_char_size(size, size, dpi, dpi) {
             Ok(_) => {
@@ -116,6 +130,13 @@ impl Face {
             }
         };
 
+        self.size.replace(FaceSize {
+            size: point_size,
+            dpi,
+            cell_width,
+            cell_height,
+        });
+
         Ok((cell_width, cell_height))
     }
 
@@ -263,6 +284,7 @@ impl Library {
                 )
             })?,
             _bytes: data,
+            size: None,
         })
     }
 
@@ -283,6 +305,7 @@ impl Library {
             face: ft_result(res, face)
                 .with_context(|| format!("FT_New_Memory_Face for index {}", face_index))?,
             _bytes: data,
+            size: None,
         })
     }
 
diff --git a/wezterm-font/src/shaper/harfbuzz.rs b/wezterm-font/src/shaper/harfbuzz.rs
index d7d465d11..be6873562 100644
--- a/wezterm-font/src/shaper/harfbuzz.rs
+++ b/wezterm-font/src/shaper/harfbuzz.rs
@@ -50,10 +50,6 @@ fn make_glyphinfo(text: &str, font_idx: usize, info: &Info) -> GlyphInfo {
 struct FontPair {
     face: ftwrap::Face,
     font: harfbuzz::Font,
-    size: f64,
-    dpi: u32,
-    cell_width: f64,
-    cell_height: f64,
 }
 
 #[derive(Debug, Hash, PartialEq, Eq, Clone)]
@@ -133,14 +129,7 @@ impl HarfbuzzShaper {
                     let mut font = harfbuzz::Font::new(face.face);
                     let (load_flags, _) = ftwrap::compute_load_flags_from_config();
                     font.set_load_flags(load_flags);
-                    *opt_pair = Some(FontPair {
-                        face,
-                        font,
-                        size: 0.,
-                        dpi: 0,
-                        cell_width: 0.,
-                        cell_height: 0.,
-                    });
+                    *opt_pair = Some(FontPair { face, font });
                 }
 
                 Ok(Some(RefMut::map(opt_pair, |opt_pair| {
@@ -180,14 +169,8 @@ impl HarfbuzzShaper {
             match self.load_fallback(font_idx)? {
                 #[allow(clippy::float_cmp)]
                 Some(mut pair) => {
-                    if pair.size != font_size || pair.dpi != dpi {
-                        let (width, height) = pair.face.set_font_size(font_size, dpi)?;
-                        pair.size = font_size;
-                        pair.dpi = dpi;
-                        pair.cell_width = width;
-                        pair.cell_height = height;
-                    }
-                    cell_width = pair.cell_width;
+                    let (width, _height) = pair.face.set_font_size(font_size, dpi)?;
+                    cell_width = width;
                     pair.font.shape(&mut buf, Some(features.as_slice()));
                 }
                 None => {