wayland: If GTK is disabled, try to avoid libdecor's GTK plugin

As described in the previous commit, loading GTK while setuid or setgid
would result in the process exiting. This is equally true if it's loaded
indirectly, for a libdecor plugin.

libdecor doesn't currently have any API by which it can be asked to
avoid specific plugins, but its GTK plugin declines to initialize if it
detects a non-main thread (because GTK documents that it must only be
used from the main thread), resulting in libdecor falling back to the
lower-priority Cairo plugin. We can make use of this by intentionally
initializing libdecor on another thread if we have been asked to avoid
GTK. This is a bit of a hack, but at worst it should be harmless.

Resolves: https://github.com/libsdl-org/sdl2-compat/issues/564
Signed-off-by: Simon McVittie <smcv@debian.org>
(cherry picked from commit 370e9407b585466b5ac54cb5240d5eb1e11fc80b)

src/video/wayland/SDL_waylandvideo.c

@@ -24,6 +24,7 @@
 #ifdef SDL_VIDEO_DRIVER_WAYLAND
 
 #include "../../core/linux/SDL_system_theme.h"
+#include "../../core/unix/SDL_gtk.h"
 #include "../../events/SDL_events_c.h"
 
 #include "SDL_waylandclipboard.h"
@@ -1380,6 +1381,19 @@ static bool should_use_libdecor(SDL_VideoData *data, bool ignore_xdg)
 
     return true;
 }
+
+static void LibdecorNew(SDL_VideoData *data)
+{
+    data->shell.libdecor = libdecor_new(data->display, &libdecor_interface);
+}
+
+// Called in another thread, but the UI thread is blocked in SDL_WaitThread
+// during that time, so it should be OK to dereference data without locks
+static int SDLCALL LibdecorNewInThread(void *data)
+{
+    LibdecorNew((SDL_VideoData *)data);
+    return 0;
+}
 #endif
 
 bool Wayland_LoadLibdecor(SDL_VideoData *data, bool ignore_xdg)
@@ -1389,7 +1403,18 @@ bool Wayland_LoadLibdecor(SDL_VideoData *data, bool ignore_xdg)
         return true; // Already loaded!
     }
     if (should_use_libdecor(data, ignore_xdg)) {
-        data->shell.libdecor = libdecor_new(data->display, &libdecor_interface);
+        if (SDL_CanUseGtk()) {
+            LibdecorNew(data);
+        } else {
+            // Intentionally initialize libdecor in a non-main thread
+            // so that it will not use its GTK plugin, but instead will
+            // fall back to the Cairo or dummy plugin
+            SDL_Thread *thread = SDL_CreateThread(LibdecorNewInThread, "SDL_LibdecorNew", (void *)data);
+            // Note that the other thread now "owns" data until we have
+            // waited for it, so don't touch data here
+            SDL_WaitThread(thread, NULL);
+        }
+
         return data->shell.libdecor != NULL;
     }
 #endif