diff --git a/wezterm-gui/src/termwindow/mod.rs b/wezterm-gui/src/termwindow/mod.rs
index 72aa6f5c0..bba5f13ac 100644
--- a/wezterm-gui/src/termwindow/mod.rs
+++ b/wezterm-gui/src/termwindow/mod.rs
@@ -229,6 +229,9 @@ pub struct TermWindow {
     tab_bar: TabBarState,
     pub right_status: String,
     last_ui_item: Option<UIItem>,
+    /// Tracks whether the current mouse-down event is part of click-focus.
+    /// If so, we ignore mouse events until released
+    is_click_to_focus: bool,
     last_mouse_coords: (usize, i64),
     last_mouse_terminal_coords: (usize, StableRowIndex),
     window_drag_position: Option<MouseEvent>,
@@ -323,6 +326,7 @@ impl TermWindow {
         if self.focused.is_none() {
             self.last_mouse_click = None;
             self.current_mouse_buttons.clear();
+            self.is_click_to_focus = false;
         }
 
         // Reset the cursor blink phase
@@ -652,6 +656,7 @@ impl TermWindow {
             ui_items: vec![],
             dragging: None,
             last_ui_item: None,
+            is_click_to_focus: false,
         };
 
         let tw = Rc::new(RefCell::new(myself));
diff --git a/wezterm-gui/src/termwindow/mouseevent.rs b/wezterm-gui/src/termwindow/mouseevent.rs
index 8fda53ed2..046731395 100644
--- a/wezterm-gui/src/termwindow/mouseevent.rs
+++ b/wezterm-gui/src/termwindow/mouseevent.rs
@@ -12,7 +12,6 @@ use std::convert::TryInto;
 use std::ops::Sub;
 use std::rc::Rc;
 use std::sync::Arc;
-use std::time::Duration;
 use wezterm_term::input::MouseEventKind as TMEK;
 use wezterm_term::{LastMouseClick, StableRowIndex};
 
@@ -471,10 +470,18 @@ impl super::TermWindow {
                 break;
             }
         }
-        if let Some(focused) = self.focused.as_ref() {
-            if focused.elapsed() <= Duration::from_millis(200) {
-                if is_click_to_focus {
-                    context.invalidate();
+        if self.focused.is_some() {
+            // Entering click-to-focus state
+            if is_click_to_focus {
+                context.invalidate();
+                self.is_click_to_focus = true;
+                return;
+            }
+
+            // Check to see if we can leave click-to-focus state
+            if self.is_click_to_focus {
+                if matches!(event.kind, WMEK::Release(_)) {
+                    self.is_click_to_focus = false;
                 }
                 return;
             }