wayland: Support xdg_decoration requesting client-side decorations.

Don't be fooled by the diff size - this ended up being a big refactor of the
shell surface management, masked only by some helper macros I wrote for the
popup support.

This change makes it so when xdg_decoration is supported, but CSD is requested,
the system bails on xdg support entirely and resets all the windows to use
libdecor instead. This transition isn't pretty, but once it's done it will be
smooth if decorations are an OS toggle since libdecor will take things from
there.

In hindsight, we really should have designed libdecor to be passed a toplevel,
having it manage that for us keeps causing major refactors for _every_ change.

src/video/wayland/SDL_waylandevents.c

@@ -476,19 +476,16 @@ ProcessHitTest(struct SDL_WaylandInput *input, uint32_t serial)
         const uint32_t *directions_libdecor = directions;
 #endif
 
-        /* Hit tests shouldn't apply to xdg_popups, right? */
-        SDL_assert(!WINDOW_IS_XDG_POPUP(window));
-
         switch (rc) {
             case SDL_HITTEST_DRAGGABLE:
 #ifdef HAVE_LIBDECOR_H
-                if (input->display->shell.libdecor) {
+                if (window_data->shell_surface_type == WAYLAND_SURFACE_LIBDECOR) {
                     if (window_data->shell_surface.libdecor.frame) {
                         libdecor_frame_move(window_data->shell_surface.libdecor.frame, input->seat, serial);
                     }
                 } else
 #endif
-                if (input->display->shell.xdg) {
+                if (window_data->shell_surface_type == WAYLAND_SURFACE_XDG_TOPLEVEL) {
                     if (window_data->shell_surface.xdg.roleobj.toplevel) {
                         xdg_toplevel_move(window_data->shell_surface.xdg.roleobj.toplevel,
                                           input->seat,
@@ -506,13 +503,13 @@ ProcessHitTest(struct SDL_WaylandInput *input, uint32_t serial)
             case SDL_HITTEST_RESIZE_BOTTOMLEFT:
             case SDL_HITTEST_RESIZE_LEFT:
 #ifdef HAVE_LIBDECOR_H
-                if (input->display->shell.libdecor) {
+                if (window_data->shell_surface_type == WAYLAND_SURFACE_LIBDECOR) {
                     if (window_data->shell_surface.libdecor.frame) {
                         libdecor_frame_resize(window_data->shell_surface.libdecor.frame, input->seat, serial, directions_libdecor[rc - SDL_HITTEST_RESIZE_TOPLEFT]);
                     }
                 } else
 #endif
-                if (input->display->shell.xdg) {
+                if (window_data->shell_surface_type == WAYLAND_SURFACE_XDG_TOPLEVEL) {
                     if (window_data->shell_surface.xdg.roleobj.toplevel) {
                         xdg_toplevel_resize(window_data->shell_surface.xdg.roleobj.toplevel,
                                             input->seat,

src/video/wayland/SDL_waylandvideo.c

@@ -878,7 +878,7 @@ static const struct wl_registry_listener registry_listener = {
 };
  
 #ifdef HAVE_LIBDECOR_H
-static SDL_bool should_use_libdecor(SDL_VideoData *data)
+static SDL_bool should_use_libdecor(SDL_VideoData *data, SDL_bool ignore_xdg)
 {
     if (!SDL_WAYLAND_HAVE_WAYLAND_LIBDECOR) {
         return SDL_FALSE;
@@ -892,6 +892,10 @@ static SDL_bool should_use_libdecor(SDL_VideoData *data)
         return SDL_TRUE;
     }
 
+    if (ignore_xdg) {
+        return SDL_TRUE;
+    }
+
     if (data->decoration_manager) {
         return SDL_FALSE;
     }
@@ -900,6 +904,18 @@ static SDL_bool should_use_libdecor(SDL_VideoData *data)
 }
 #endif
 
+SDL_bool
+Wayland_LoadLibdecor(SDL_VideoData *data, SDL_bool ignore_xdg)
+{
+#ifdef HAVE_LIBDECOR_H
+    if (should_use_libdecor(data, ignore_xdg)) {
+        data->shell.libdecor = libdecor_new(data->display, &libdecor_interface);
+        return data->shell.libdecor != NULL;
+    }
+#endif
+    return SDL_FALSE;
+}
+
 int
 Wayland_VideoInit(_THIS)
 {
@@ -920,12 +936,8 @@ Wayland_VideoInit(_THIS)
     // First roundtrip to receive all registry objects.
     WAYLAND_wl_display_roundtrip(data->display);
 
-#ifdef HAVE_LIBDECOR_H
+    /* Now that we have all the protocols, load libdecor if applicable */
-    /* Don't have server-side decorations? Try client-side instead. */
+    Wayland_LoadLibdecor(data, SDL_FALSE);
-    if (should_use_libdecor(data)) {
-        data->shell.libdecor = libdecor_new(data->display, &libdecor_interface);
-    }
-#endif
 
     // Second roundtrip to receive all output events.
     WAYLAND_wl_display_roundtrip(data->display);

src/video/wayland/SDL_waylandvideo.h

@@ -122,6 +122,8 @@ extern void SDL_WAYLAND_register_output(struct wl_output *output);
 extern SDL_bool SDL_WAYLAND_own_surface(struct wl_surface *surface);
 extern SDL_bool SDL_WAYLAND_own_output(struct wl_output *output);
 
+extern SDL_bool Wayland_LoadLibdecor(SDL_VideoData *data, SDL_bool ignore_xdg);
+
 #endif /* SDL_waylandvideo_h_ */
 
 /* vi: set ts=4 sw=4 expandtab: */

src/video/wayland/SDL_waylandwindow.c

@@ -683,6 +683,34 @@ Wayland_PopupWatch(void *data, SDL_Event *event)
     return 1;
 }
 
+static void
+handle_configure_zxdg_decoration(void *data,
+                                 struct zxdg_toplevel_decoration_v1 *zxdg_toplevel_decoration_v1,
+                                 uint32_t mode)
+{
+    SDL_Window *window = (SDL_Window *) data;
+    SDL_WindowData *driverdata = (SDL_WindowData *) window->driverdata;
+
+    /* If the compositor tries to force CSD anyway, bail on direct XDG support
+     * and fall back to libdecor, it will handle these events from then on.
+     *
+     * To do this we have to fully unmap, then map with libdecor loaded.
+     */
+    if (mode == ZXDG_TOPLEVEL_DECORATION_V1_MODE_CLIENT_SIDE) {
+        if (!Wayland_LoadLibdecor(driverdata->waylandData, SDL_TRUE)) {
+            /* libdecor isn't available, so no borders for you... oh well */
+            return;
+        }
+        SDL_HideWindow(window);
+        driverdata->shell_surface_type = WAYLAND_SURFACE_LIBDECOR;
+        SDL_ShowWindow(window);
+    }
+}
+
+static const struct zxdg_toplevel_decoration_v1_listener decoration_listener = {
+    handle_configure_zxdg_decoration
+};
+
 #ifdef HAVE_LIBDECOR_H
 static void
 decoration_frame_configure(struct libdecor_frame *frame,
@@ -1258,6 +1286,9 @@ void Wayland_ShowWindow(_THIS, SDL_Window *window)
         /* Create the window decorations */
         if (!WINDOW_IS_XDG_POPUP(window) && data->shell_surface.xdg.roleobj.toplevel && c->decoration_manager) {
             data->server_decoration = zxdg_decoration_manager_v1_get_toplevel_decoration(c->decoration_manager, data->shell_surface.xdg.roleobj.toplevel);
+            zxdg_toplevel_decoration_v1_add_listener(data->server_decoration,
+                                                     &decoration_listener,
+                                                     window);
         }
     } else {
         /* Nothing to see here, just commit. */
@@ -1628,7 +1659,6 @@ void
 Wayland_SetWindowResizable(_THIS, SDL_Window * window, SDL_bool resizable)
 {
 #ifdef HAVE_LIBDECOR_H
-    SDL_VideoData *data = _this->driverdata;
     const SDL_WindowData *wind = window->driverdata;
 
     if (WINDOW_IS_LIBDECOR(data, window)) {
@@ -1874,6 +1904,19 @@ int Wayland_CreateWindow(_THIS, SDL_Window *window)
     /* We may need to create an idle inhibitor for this new window */
     Wayland_SuspendScreenSaver(_this);
 
+    #define IS_POPUP(window) \
+        (window->flags & (SDL_WINDOW_TOOLTIP | SDL_WINDOW_POPUP_MENU))
+    if (c->shell.libdecor && !IS_POPUP(window)) {
+        data->shell_surface_type = WAYLAND_SURFACE_LIBDECOR;
+    } else if (c->shell.xdg) {
+        if (IS_POPUP(window)) {
+            data->shell_surface_type = WAYLAND_SURFACE_XDG_POPUP;
+        } else {
+            data->shell_surface_type = WAYLAND_SURFACE_XDG_TOPLEVEL;
+        }
+    } /* All other cases will be WAYLAND_SURFACE_UNKNOWN */
+    #undef IS_POPUP
+
     return 0;
 }
 

src/video/wayland/SDL_waylandwindow.h

@@ -32,7 +32,28 @@
 
 struct SDL_WaylandInput;
 
+/* TODO: Remove these helpers, they're from before we had shell_surface_type */
+#define WINDOW_IS_XDG_POPUP(window) \
+    (((SDL_WindowData*) window->driverdata)->shell_surface_type == WAYLAND_SURFACE_XDG_POPUP)
+#define WINDOW_IS_LIBDECOR(ignoreme, window) \
+    (((SDL_WindowData*) window->driverdata)->shell_surface_type == WAYLAND_SURFACE_LIBDECOR)
+
 typedef struct {
+    SDL_Window *sdlwindow;
+    SDL_VideoData *waylandData;
+    struct wl_surface *surface;
+    struct wl_callback *frame_callback;
+    struct wl_event_queue *frame_event_queue;
+    struct wl_surface *frame_surface_wrapper;
+
+    union {
+#ifdef HAVE_LIBDECOR_H
+        struct {
+            struct libdecor_frame *frame;
+            SDL_bool initial_configure_seen;
+        } libdecor;
+#endif
+        struct {
             struct xdg_surface *surface;
             union {
                 struct xdg_toplevel *toplevel;
@@ -44,34 +65,15 @@ typedef struct {
                 } popup;
             } roleobj;
             SDL_bool initial_configure_seen;
-} SDL_xdg_shell_surface;
+        } xdg;
-
-#define WINDOW_IS_XDG_POPUP(window) \
-    (window->flags & (SDL_WINDOW_TOOLTIP | SDL_WINDOW_POPUP_MENU))
-
-#ifdef HAVE_LIBDECOR_H
-typedef struct {
-    struct libdecor_frame *frame;
-    SDL_bool initial_configure_seen;
-} SDL_libdecor_surface;
-
-#define WINDOW_IS_LIBDECOR(viddata, window) \
-    (viddata->shell.libdecor && !WINDOW_IS_XDG_POPUP(window))
-#endif
-
-typedef struct {
-    SDL_Window *sdlwindow;
-    SDL_VideoData *waylandData;
-    struct wl_surface *surface;
-    struct wl_callback *frame_callback;
-    struct wl_event_queue *frame_event_queue;
-    struct wl_surface *frame_surface_wrapper;
-    union {
-#ifdef HAVE_LIBDECOR_H
-        SDL_libdecor_surface libdecor;
-#endif
-        SDL_xdg_shell_surface xdg;
     } shell_surface;
+    enum {
+        WAYLAND_SURFACE_UNKNOWN = 0,
+        WAYLAND_SURFACE_XDG_TOPLEVEL,
+        WAYLAND_SURFACE_XDG_POPUP,
+        WAYLAND_SURFACE_LIBDECOR
+    } shell_surface_type;
+
     struct wl_egl_window *egl_window;
     struct SDL_WaylandInput *keyboard_device;
 #if SDL_VIDEO_OPENGL_EGL