diff --git a/wezterm-font/src/ftwrap.rs b/wezterm-font/src/ftwrap.rs
index 803d498b7..e36562314 100644
--- a/wezterm-font/src/ftwrap.rs
+++ b/wezterm-font/src/ftwrap.rs
@@ -174,6 +174,18 @@ impl Face {
         }
     }
 
+    /// Returns the cap_height/units_per_EM ratio if known
+    pub fn cap_height(&self) -> Option<f64> {
+        unsafe {
+            let os2 = self.get_os2_table()?;
+            let units_per_em = (*self.face).units_per_EM;
+            if units_per_em == 0 || os2.sCapHeight == 0 {
+                return None;
+            }
+            Some(os2.sCapHeight as f64 / units_per_em as f64)
+        }
+    }
+
     pub fn weight_and_width(&self) -> (u16, u16) {
         let (mut weight, mut width) = self
             .get_os2_table()
diff --git a/wezterm-font/src/parser.rs b/wezterm-font/src/parser.rs
index cb310211b..84bd0dedc 100644
--- a/wezterm-font/src/parser.rs
+++ b/wezterm-font/src/parser.rs
@@ -18,6 +18,7 @@ pub struct ParsedFont {
     weight: FontWeight,
     stretch: FontStretch,
     italic: bool,
+    cap_height: Option<f64>,
     pub handle: FontDataHandle,
     coverage: Mutex<RangeSet<u32>>,
 }
@@ -30,6 +31,7 @@ impl std::fmt::Debug for ParsedFont {
             .field("stretch", &self.stretch)
             .field("italic", &self.italic)
             .field("handle", &self.handle)
+            .field("cap_height", &self.cap_height)
             .finish()
     }
 }
@@ -42,6 +44,7 @@ impl Clone for ParsedFont {
             stretch: self.stretch,
             italic: self.italic,
             handle: self.handle.clone(),
+            cap_height: self.cap_height.clone(),
             coverage: Mutex::new(self.coverage.lock().unwrap().clone()),
         }
     }
@@ -127,6 +130,7 @@ impl ParsedFont {
         let (weight, width) = face.weight_and_width();
         let weight = FontWeight::from_opentype_weight(weight);
         let stretch = FontStretch::from_opentype_stretch(width);
+        let cap_height = face.cap_height();
 
         Ok(Self {
             names: Names::from_ft_face(&face),
@@ -135,6 +139,7 @@ impl ParsedFont {
             italic,
             handle,
             coverage: Mutex::new(RangeSet::new()),
+            cap_height,
         })
     }
 
diff --git a/wezterm-font/src/shaper/allsorts.rs b/wezterm-font/src/shaper/allsorts.rs
index 4455495fd..0fbf21571 100644
--- a/wezterm-font/src/shaper/allsorts.rs
+++ b/wezterm-font/src/shaper/allsorts.rs
@@ -226,6 +226,7 @@ impl AllsortsParsedFont {
             descender,
             underline_thickness,
             underline_position,
+            cap_height_ratio: None,
         };
 
         log::trace!("metrics: {:?}", metrics);
diff --git a/wezterm-font/src/shaper/harfbuzz.rs b/wezterm-font/src/shaper/harfbuzz.rs
index ebcab4a4a..d945a4873 100644
--- a/wezterm-font/src/shaper/harfbuzz.rs
+++ b/wezterm-font/src/shaper/harfbuzz.rs
@@ -411,6 +411,7 @@ impl FontShaper for HarfbuzzShaper {
             underline_position: PixelLength::new(
                 unsafe { (*pair.face.face).underline_position as f64 } * y_scale / 64.,
             ),
+            cap_height_ratio: pair.face.cap_height(),
         };
 
         self.metrics.borrow_mut().insert(key, metrics.clone());
diff --git a/wezterm-font/src/shaper/mod.rs b/wezterm-font/src/shaper/mod.rs
index 374d1a69a..5e3d55498 100644
--- a/wezterm-font/src/shaper/mod.rs
+++ b/wezterm-font/src/shaper/mod.rs
@@ -51,6 +51,9 @@ pub struct FontMetrics {
     /// Position of underline relative to descender. Negative
     /// values are below the descender.
     pub underline_position: PixelLength,
+
+    /// Fraction of the EM square occupied by the cap height
+    pub cap_height_ratio: Option<f64>,
 }
 
 pub trait FontShaper {
diff --git a/wezterm-gui/src/glyphcache.rs b/wezterm-gui/src/glyphcache.rs
index e48d92833..ec182b168 100644
--- a/wezterm-gui/src/glyphcache.rs
+++ b/wezterm-gui/src/glyphcache.rs
@@ -465,11 +465,33 @@ impl<T: Texture2d> GlyphCache<T> {
             idx_metrics = font.metrics_for_idx(info.font_idx)?;
         }
 
-        let y_scale = base_metrics.cell_height.get() / idx_metrics.cell_height.get();
-        let x_scale =
-            base_metrics.cell_width.get() / (idx_metrics.cell_width.get() / info.num_cells as f64);
+        let x_scale;
+        let y_scale;
 
-        let aspect = (idx_metrics.cell_height / idx_metrics.cell_width).get();
+        if info.font_idx == 0 {
+            // The base font is the current font, so there's no additional metrics
+            // based scaling, however, we may need to scale to accomodate num_cells
+            x_scale = 1.0 / info.num_cells as f64;
+            y_scale = 1.0;
+        } else if let (Some(base_cap), Some(cap)) =
+            (base_metrics.cap_height_ratio, idx_metrics.cap_height_ratio)
+        {
+            // If we know the cap height ratio for both fonts, we can scale
+            // the second one to match the cap height of the first
+            x_scale = base_cap / cap;
+            y_scale = x_scale / info.num_cells as f64;
+        } else {
+            // Otherwise, we scale based on the ratio of the metrics for
+            // the two fonts.
+            // If we know the cap height ratio for the first, we can adjust
+            // the overall scale so that the second font isn't oversized
+            let base_cap = base_metrics.cap_height_ratio.unwrap_or(1.);
+            y_scale = base_cap * base_metrics.cell_height.get() / idx_metrics.cell_height.get();
+            x_scale = base_cap * base_metrics.cell_width.get()
+                / (idx_metrics.cell_width.get() / info.num_cells as f64);
+        }
+
+        let aspect = (idx_metrics.cell_width / idx_metrics.cell_height).get();
         let is_square_or_wide = aspect >= 0.9;
 
         let allow_width_overflow = if is_square_or_wide {