diff --git a/src/x11/events.c b/src/x11/events.c
index 03179e4af..f4a279d9b 100644
--- a/src/x11/events.c
+++ b/src/x11/events.c
@@ -907,12 +907,11 @@ handle_window_focus_event (MetaX11Display *x11_display,
       (!x11_display->focused_by_us &&
        x11_display->server_focus_serial == x11_display->focus_serial))
     {
-      meta_x11_display_update_focus_window (x11_display,
-                                            focus_window ?
-                                            meta_window_x11_get_xwindow (focus_window) : None,
-                                            x11_display->server_focus_serial,
-                                            FALSE);
-      meta_display_update_focus_window (display, focus_window);
+      x11_display->is_server_focus = TRUE;
+      meta_display_set_input_focus (display, focus_window,
+                                    meta_display_get_current_time_roundtrip (display));
+      x11_display->is_server_focus = FALSE;
+
       return TRUE;
     }
   else
@@ -1895,13 +1894,11 @@ meta_x11_display_handle_xevent (MetaX11Display *x11_display,
     {
       meta_topic (META_DEBUG_FOCUS, "Earlier attempt to focus %s failed",
                   display->focus_window->desc);
-      meta_x11_display_update_focus_window (x11_display,
-                                            x11_display->server_focus_window,
-                                            x11_display->server_focus_serial,
-                                            FALSE);
-      meta_display_update_focus_window (display,
-                                        meta_x11_display_lookup_x_window (x11_display,
-                                                                          x11_display->server_focus_window));
+
+      x11_display->is_server_focus = TRUE;
+      meta_display_set_input_focus (display, display->focus_window,
+                                    meta_display_get_current_time_roundtrip (display));
+      x11_display->is_server_focus = FALSE;
     }
 
   if (event->xany.window == x11_display->xroot)
diff --git a/src/x11/meta-x11-display-private.h b/src/x11/meta-x11-display-private.h
index 1c3dc7d2b..1f47849b7 100644
--- a/src/x11/meta-x11-display-private.h
+++ b/src/x11/meta-x11-display-private.h
@@ -152,6 +152,8 @@ struct _MetaX11Display
    */
   guint focused_by_us : 1;
 
+  guint is_server_focus : 1;
+
   guint keys_grabbed : 1;
 
   guint closing : 1;
diff --git a/src/x11/meta-x11-display.c b/src/x11/meta-x11-display.c
index 40339cef8..438f3bd1e 100644
--- a/src/x11/meta-x11-display.c
+++ b/src/x11/meta-x11-display.c
@@ -2118,11 +2118,20 @@ meta_x11_display_set_input_focus (MetaX11Display *x11_display,
   meta_topic (META_DEBUG_FOCUS, "Setting X11 input focus for window %s to 0x%lx",
               window ? window->desc : "none", xwindow);
 
-  mtk_x11_error_trap_push (x11_display->xdisplay);
-  meta_x11_display_set_input_focus_internal (x11_display, xwindow, timestamp);
-  serial = XNextRequest (x11_display->xdisplay);
-  meta_x11_display_update_focus_window (x11_display, xwindow, serial, TRUE);
-  mtk_x11_error_trap_pop (x11_display->xdisplay);
+  if (x11_display->is_server_focus)
+    {
+      serial = x11_display->server_focus_serial;
+    }
+  else
+    {
+      meta_x11_display_set_input_focus_internal (x11_display, xwindow, timestamp);
+      mtk_x11_error_trap_push (x11_display->xdisplay);
+      serial = XNextRequest (x11_display->xdisplay);
+      mtk_x11_error_trap_pop (x11_display->xdisplay);
+    }
+
+  meta_x11_display_update_focus_window (x11_display, xwindow, serial,
+                                        !x11_display->is_server_focus);
 
 #ifdef HAVE_X11
   if (window && !meta_is_wayland_compositor ())