diff --git a/src/video/SDL_video.c b/src/video/SDL_video.c
index 32baa6efd0..3e926648be 100644
--- a/src/video/SDL_video.c
+++ b/src/video/SDL_video.c
@@ -1256,9 +1256,6 @@ SDL_DisplayID SDL_GetDisplayForWindow(SDL_Window *window)
      * (for example if the window is off-screen), but other code may expect it
      * to succeed in that situation, so we fall back to a generic position-
      * based implementation in that case. */
-    if (!displayID && (window->flags & SDL_WINDOW_FULLSCREEN)) {
-        displayID = window->fullscreen_mode.displayID;
-    }
     if (!displayID) {
         displayID = SDL_GetDisplayForWindowCoordinate(window->windowed.x);
     }
@@ -1267,10 +1264,14 @@ SDL_DisplayID SDL_GetDisplayForWindow(SDL_Window *window)
     }
     if (!displayID) {
         displayID = GetDisplayForRect(window->x, window->y, window->w, window->h);
-        if (!displayID) {
-            /* Use the primary display for a window if we can't find it anywhere else */
-            displayID = SDL_GetPrimaryDisplay();
-        }
+    }
+    if (!displayID && (window->flags & SDL_WINDOW_FULLSCREEN)) {
+        /* Use the explicit fullscreen display if retrieval via the window position fails */
+        displayID = window->fullscreen_mode.displayID;
+    }
+    if (!displayID) {
+        /* Use the primary display for a window if we can't find it anywhere else */
+        displayID = SDL_GetPrimaryDisplay();
     }
     return displayID;
 }
@@ -1514,8 +1515,16 @@ int SDL_SetWindowFullscreenMode(SDL_Window *window, const SDL_DisplayMode *mode)
 
     if (SDL_WINDOW_FULLSCREEN_VISIBLE(window)) {
         SDL_UpdateFullscreenMode(window, SDL_TRUE);
-    } else {
-        SDL_CheckWindowDisplayChanged(window);
+    } else if (window->flags & SDL_WINDOW_FULLSCREEN) {
+        /* If fullscreen and not visible, just update the position so the window will be
+         * on the correct display when shown/restored.
+         */
+        if (mode) {
+            SDL_Rect r;
+            if (SDL_GetDisplayBounds(mode->displayID, &r) == 0) {
+                SDL_SendWindowEvent(window, SDL_EVENT_WINDOW_MOVED, r.x, r.y);
+            }
+        }
     }
     return 0;
 }