Handle interaction between auto capture and the SDL_CaptureMouse() API

Fixes https://github.com/libsdl-org/SDL/issues/5457

src/events/SDL_keyboard.c

@@ -638,6 +638,7 @@ SDL_SetKeyboardFocus(SDL_Window * window)
         /* old window must lose an existing mouse capture. */
         if (keyboard->focus->flags & SDL_WINDOW_MOUSE_CAPTURE) {
             SDL_CaptureMouse(SDL_FALSE);  /* drop the capture. */
+            SDL_UpdateMouseCapture(SDL_TRUE);
             SDL_assert(!(keyboard->focus->flags & SDL_WINDOW_MOUSE_CAPTURE));
         }
 

src/events/SDL_mouse.c

@@ -156,12 +156,8 @@ SDL_MouseAutoCaptureChanged(void *userdata, const char *name, const char *oldVal
     SDL_bool auto_capture = SDL_GetStringBoolean(hint, SDL_TRUE);
 
     if (auto_capture != mouse->auto_capture) {
-        /* Turn off mouse capture if it's currently active because of button presses */
-        if (!auto_capture && SDL_GetMouseState(NULL, NULL) != 0) {
-            SDL_CaptureMouse(SDL_FALSE);
-        }
-
         mouse->auto_capture = auto_capture;
+        SDL_UpdateMouseCapture(SDL_FALSE);
     }
 }
 
@@ -668,10 +664,7 @@ SDL_PrivateSendMouseButton(SDL_Window * window, SDL_MouseID mouseID, Uint8 state
 
     /* Automatically capture the mouse while buttons are pressed */
     if (mouse->auto_capture) {
-        SDL_bool has_buttons_pressed = (SDL_GetMouseState(NULL, NULL) ? SDL_TRUE : SDL_FALSE);
+        SDL_UpdateMouseCapture(SDL_FALSE);
-        if (has_buttons_pressed != had_buttons_pressed) {
-            SDL_CaptureMouse(has_buttons_pressed);
-        }
     }
 
     return posted;
@@ -768,6 +761,7 @@ SDL_MouseQuit(void)
 
     if (mouse->CaptureMouse) {
         SDL_CaptureMouse(SDL_FALSE);
+        SDL_UpdateMouseCapture(SDL_TRUE);
     }
     SDL_SetRelativeMouseMode(SDL_FALSE);
     SDL_ShowCursor(1);
@@ -972,6 +966,8 @@ SDL_SetRelativeMouseMode(SDL_bool enabled)
         if (!enabled) {
             SDL_WarpMouseInWindow(focusWindow, mouse->x, mouse->y);
         }
+
+        SDL_UpdateMouseCapture(SDL_FALSE);
     }
 
     if (!enabled) {
@@ -993,39 +989,61 @@ SDL_GetRelativeMouseMode()
     return mouse->relative_mode;
 }
 
+int
+SDL_UpdateMouseCapture(SDL_bool force_release)
+{
+    SDL_Mouse *mouse = SDL_GetMouse();
+    SDL_Window *capture_window = NULL;
+
+    if (!mouse->CaptureMouse) {
+        return 0;
+    }
+
+    if (!force_release) {
+        if (mouse->capture_desired || (mouse->auto_capture && SDL_GetMouseState(NULL, NULL) != 0)) {
+            if (!mouse->relative_mode) {
+                capture_window = SDL_GetKeyboardFocus();
+            }
+        }
+    }
+
+    if (capture_window != mouse->capture_window) {
+        if (mouse->capture_window) {
+            mouse->CaptureMouse(NULL);
+            mouse->capture_window->flags &= ~SDL_WINDOW_MOUSE_CAPTURE;
+            mouse->capture_window = NULL;
+        }
+
+        if (capture_window) {
+            if (mouse->CaptureMouse(capture_window) < 0) {
+                /* CaptureMouse() will have set an error */
+                return -1;
+            }
+            capture_window->flags |= SDL_WINDOW_MOUSE_CAPTURE;
+        }
+
+        mouse->capture_window = capture_window;
+    }
+    return 0;
+}
+
 int
 SDL_CaptureMouse(SDL_bool enabled)
 {
     SDL_Mouse *mouse = SDL_GetMouse();
-    SDL_Window *focusWindow;
+    SDL_Window *focus_window;
-    SDL_bool isCaptured;
 
     if (!mouse->CaptureMouse) {
         return SDL_Unsupported();
     }
 
-    focusWindow = SDL_GetKeyboardFocus();
+    focus_window = SDL_GetKeyboardFocus();
-
+    if (enabled && !focus_window) {
-    isCaptured = focusWindow && (focusWindow->flags & SDL_WINDOW_MOUSE_CAPTURE);
-    if (isCaptured == enabled) {
-        return 0;  /* already done! */
-    }
-
-    if (enabled) {
-        if (!focusWindow) {
         return SDL_SetError("No window has focus");
-        } else if (mouse->CaptureMouse(focusWindow) == -1) {
-            return -1;  /* CaptureMouse() should call SetError */
-        }
-        focusWindow->flags |= SDL_WINDOW_MOUSE_CAPTURE;
-    } else {
-        if (mouse->CaptureMouse(NULL) == -1) {
-            return -1;  /* CaptureMouse() should call SetError */
-        }
-        focusWindow->flags &= ~SDL_WINDOW_MOUSE_CAPTURE;
     }
+    mouse->capture_desired = enabled;
 
-    return 0;
+    return SDL_UpdateMouseCapture(SDL_FALSE);
 }
 
 SDL_Cursor *

src/events/SDL_mouse_c.h

@@ -104,6 +104,8 @@ typedef struct
     Uint8 vita_touch_mouse_device;
 #endif
     SDL_bool auto_capture;
+    SDL_bool capture_desired;
+    SDL_Window *capture_window;
 
     /* Data for input source state */
     int num_sources;
@@ -135,6 +137,9 @@ extern void SDL_SetDefaultCursor(SDL_Cursor * cursor);
 /* Set the mouse focus window */
 extern void SDL_SetMouseFocus(SDL_Window * window);
 
+/* Update the mouse capture window */
+extern int SDL_UpdateMouseCapture(SDL_bool force_release);
+
 /* Send a mouse motion event */
 extern int SDL_SendMouseMotion(SDL_Window * window, SDL_MouseID mouseID, int relative, int x, int y);