diff --git a/wezterm-gui/src/termwindow/mouseevent.rs b/wezterm-gui/src/termwindow/mouseevent.rs
index d70c452b4..88735f236 100644
--- a/wezterm-gui/src/termwindow/mouseevent.rs
+++ b/wezterm-gui/src/termwindow/mouseevent.rs
@@ -59,13 +59,15 @@ impl super::TermWindow {
         }
     }
 
-    pub fn mouse_event_impl(&mut self, event: MouseEvent, context: &dyn WindowOps) {
+    pub fn mouse_event_impl(&mut self, mut event: MouseEvent, context: &dyn WindowOps) {
         log::trace!("{:?}", event);
         let pane = match self.get_active_pane_or_overlay() {
             Some(pane) => pane,
             None => return,
         };
 
+        event.modifiers = event.modifiers.remove_keyboard_status_mods();
+
         self.current_mouse_event.replace(event.clone());
 
         let border = self.get_os_border();