xwayland: Prevent Xserver grabs with rootless

Because of the design of most Wayland compositors, where the compositor
is both a Wayland server and an X11 window manager, any X11 client
issuing a server grab (i.e. XGrabServer()) can possibly hang the whole
desktop when Xwayland is running rootless.

This can happen with e.g. ImageMagick's import command with mutter.

1. "import" is launched and issues an XServerGrab(),
2. Xwayland restricts access to that "import" X11 client alone,
3. mutter continues to process events until it needs to sync with
   Xwayland (there's variability in time before the hang occurs),
4. When mutter does an XSync() (explicitly or implicitly through some
   other Xlib call), it will stop waiting for Xwayland to reply,
5. Xwayland waits for the XServerGrab() to be released by import,
6. "import" waits for a user input to release the XServerGrab(),
7. mutter is stuck waiting on Xwayland and does not process input
   events...

To prevent this, re-route the GrabServer/UngrabServer requests and
pretend the grab works but actually does nothing at all for all clients
but the X11 window manager (which can still issue X11 server grabs, at
its own risks).

Signed-off-by: Olivier Fourdan <ofourdan@redhat.com>
Reviewed-by: Peter Hutterer <peter.hutterer@who-t.net>
Closes: https://bugzilla.redhat.com/1914021

hw/xwayland/xwayland-screen.c

@@ -643,6 +643,52 @@ xwl_screen_roundtrip(struct xwl_screen *xwl_screen)
         xwl_give_up("could not connect to wayland server\n");
 }
 
+static int
+xwl_server_grab(ClientPtr client)
+{
+    struct xwl_screen *xwl_screen;
+
+    /* Allow GrabServer for the X11 window manager.
+     * Xwayland only has 1 screen (no Zaphod for Xwayland) so we check
+     * for the first and only screen here.
+     */
+    xwl_screen = xwl_screen_get(screenInfo.screens[0]);
+    if (xwl_screen->wm_client_id == client->index)
+        return xwl_screen->GrabServer(client);
+
+    /* For all other clients, just pretend it works for compatibility,
+       but do nothing */
+    return Success;
+}
+
+static int
+xwl_server_ungrab(ClientPtr client)
+{
+    struct xwl_screen *xwl_screen;
+
+    /* Same as above, allow UngrabServer for the X11 window manager only */
+    xwl_screen = xwl_screen_get(screenInfo.screens[0]);
+    if (xwl_screen->wm_client_id == client->index)
+        return xwl_screen->UngrabServer(client);
+
+    /* For all other clients, just pretend it works for compatibility,
+       but do nothing */
+    return Success;
+}
+
+static void
+xwl_screen_setup_custom_vector(struct xwl_screen *xwl_screen)
+{
+    /* Rootfull Xwayland does not need a custom ProcVector (yet?) */
+    if (xwl_screen->rootless)
+        return;
+
+    xwl_screen->GrabServer = ProcVector[X_GrabServer];
+    xwl_screen->UngrabServer = ProcVector[X_UngrabServer];
+
+    ProcVector[X_GrabServer] = xwl_server_grab;
+    ProcVector[X_UngrabServer] = xwl_server_ungrab;
+}
 
 int
 xwl_screen_get_next_output_serial(struct xwl_screen *xwl_screen)
@@ -932,6 +978,8 @@ xwl_screen_init(ScreenPtr pScreen, int argc, char **argv)
     AddCallback(&PropertyStateCallback, xwl_property_callback, pScreen);
     AddCallback(&RootWindowFinalizeCallback, xwl_root_window_finalized_callback, pScreen);
 
+    xwl_screen_setup_custom_vector(xwl_screen);
+
     xwl_screen_roundtrip(xwl_screen);
 
     return ret;

hw/xwayland/xwayland-screen.h

@@ -78,6 +78,9 @@ struct xwl_screen {
     ResizeWindowProcPtr ResizeWindow;
     MoveWindowProcPtr MoveWindow;
 
+    int (*GrabServer) (ClientPtr client);
+    int (*UngrabServer) (ClientPtr client);
+
     struct xorg_list output_list;
     struct xorg_list seat_list;
     struct xorg_list damage_window_list;