diff --git a/glfw/xkb_glfw.c b/glfw/xkb_glfw.c
index a76ed67df..8a256ec14 100644
--- a/glfw/xkb_glfw.c
+++ b/glfw/xkb_glfw.c
@@ -292,6 +292,9 @@ glfw_xkb_update_modifiers(_GLFWXKBData *xkb, xkb_mod_mask_t depressed, xkb_mod_m
     if (!xkb->keymap) return;
     xkb->modifiers = 0;
     xkb_state_update_mask(xkb->state, depressed, latched, locked, base_group, latched_group, locked_group);
+    // We have to update the groups in clean_state, as they change for
+    // different keyboard layouts, see https://github.com/kovidgoyal/kitty/issues/488
+    xkb_state_update_mask(xkb->clean_state, 0, 0, 0, base_group, latched_group, locked_group);
 #define S(attr, name) if (xkb_state_mod_index_is_active(xkb->state, xkb->attr##Idx, XKB_STATE_MODS_EFFECTIVE)) xkb->modifiers |= GLFW_MOD_##name
     S(control, CONTROL); S(alt, ALT); S(shift, SHIFT); S(super, SUPER); S(capsLock, CAPS_LOCK); S(numLock, NUM_LOCK);
 #undef S