unix: If setuid or setgid, don't use GTK

GTK explicitly doesn't support being used setuid or setgid, and if SDL
loads and initializes GTK, GTK will exit the process if it
detects a setgid executable. This is incompatible with the historical
practice of making game executables setgid in order to write out a
shared high-score table on multi-user systems (which is security theatre
at best, because typical game runtime libraries are not hardened against
an untrusted caller, but making it regress would be a user-observable
regression in sdl2-compat).

Helps: https://github.com/libsdl-org/sdl2-compat/issues/564
Signed-off-by: Simon McVittie <smcv@debian.org>

CMakeLists.txt

@@ -1161,6 +1161,8 @@ if(SDL_LIBC)
     check_symbol_exists(fdatasync "unistd.h" HAVE_FDATASYNC)
     check_symbol_exists(gethostname "unistd.h" HAVE_GETHOSTNAME)
     check_symbol_exists(getpagesize "unistd.h" HAVE_GETPAGESIZE)
+    check_symbol_exists(getresgid "unistd.h" HAVE_GETRESGID)
+    check_symbol_exists(getresuid "unistd.h" HAVE_GETRESUID)
     check_symbol_exists(sigaction "signal.h" HAVE_SIGACTION)
     check_symbol_exists(sigtimedwait "signal.h" HAVE_SIGTIMEDWAIT)
     check_symbol_exists(setjmp "setjmp.h" HAVE_SETJMP)

include/build_config/SDL_build_config.h.cmake

@@ -201,6 +201,8 @@
 #cmakedefine HAVE_ELF_AUX_INFO 1
 #cmakedefine HAVE_PPOLL 1
 #cmakedefine HAVE__EXIT 1
+#cmakedefine HAVE_GETRESUID 1
+#cmakedefine HAVE_GETRESGID 1
 
 #endif /* HAVE_LIBC */
 

src/core/unix/SDL_gtk.c

@@ -22,6 +22,8 @@
 #include "SDL_gtk.h"
 
 #include <dlfcn.h>
+#include <errno.h>
+#include <unistd.h>
 
 #define SDL_GTK_SYM2_OPTIONAL(ctx, lib, sub, fn, sym)                     \
     ctx.sub.fn = (void *)SDL_LoadFunction(lib, #sym)
@@ -81,9 +83,63 @@ static bool IsGtkInit()
     return libgdk != NULL && libgtk != NULL;
 }
 
+#ifndef HAVE_GETRESUID
+// Non-POSIX, but Linux and some BSDs have it.
+// To reduce the number of code paths, if getresuid() isn't available at
+// compile-time, we behave as though it existed but failed at runtime.
+static inline int getresuid(uid_t *ruid, uid_t *euid, uid_t *suid) {
+    errno = ENOSYS;
+    return -1;
+}
+#endif
+
+#ifndef HAVE_GETRESGID
+// Same as getresuid() but for the primary group
+static inline int getresgid(uid_t *ruid, uid_t *euid, uid_t *suid) {
+    errno = ENOSYS;
+    return -1;
+}
+#endif
+
 bool SDL_CanUseGtk(void)
 {
+    // "Real", "effective" and "saved" IDs: see e.g. Linux credentials(7)
+    uid_t ruid = -1, euid = -1, suid = -1;
+    gid_t rgid = -1, egid = -1, sgid = -1;
+
     if (!SDL_GetHintBoolean("SDL_ENABLE_GTK", true)) {
+        SDL_LogDebug(SDL_LOG_CATEGORY_SYSTEM, "Not using GTK due to hint");
+        return false;
+    }
+
+    // This is intended to match the check in gtkmain.c, rather than being
+    // an exhaustive check for having elevated privileges: as a result
+    // we don't use Linux getauxval() or prctl PR_GET_DUMPABLE,
+    // BSD issetugid(), or similar OS-specific detection
+
+    if (getresuid(&ruid, &euid, &suid) != 0) {
+        ruid = suid = getuid();
+        euid = geteuid();
+    }
+
+    if (getresgid(&rgid, &egid, &sgid) != 0) {
+        rgid = sgid = getgid();
+        egid = getegid();
+    }
+
+    // Real ID != effective ID means we are setuid or setgid:
+    // GTK will refuse to initialize, and instead will call exit().
+    if (ruid != euid || rgid != egid) {
+        SDL_LogDebug(SDL_LOG_CATEGORY_SYSTEM, "Not using GTK due to setuid/setgid");
+        return false;
+    }
+
+    // Real ID != saved ID means we are setuid or setgid, we previously
+    // dropped privileges, but we can regain them; this protects against
+    // accidents but does not protect against arbitrary code execution.
+    // Again, GTK will refuse to initialize if this is the case.
+    if (ruid != suid || rgid != sgid) {
+        SDL_LogDebug(SDL_LOG_CATEGORY_SYSTEM, "Not using GTK due to saved uid/gid");
         return false;
     }