wayland: Handle min/max sizes in fixed-size windows with viewports

Wayland is sometimes at-odds with clients that want to enforce an aspect ratio or min/max window size, as certain window states have dimensions that either must be obeyed (maximized), or will give terrible results if they aren't (tiled). Use a viewport and a masking subsurface to handle cases where surfaces are unable to match the exact window size.

The changes made to accommodate this also catches some additional windowing related edge-cases, simplifies synchronization, and prevents commits before a buffer has been attached to the surface.

src/video/wayland/SDL_waylandevents.c

@@ -802,7 +802,20 @@ static void pointer_handle_motion(void *data, struct wl_pointer *pointer,
 
 static void pointer_dispatch_enter(SDL_WaylandSeat *seat)
 {
-    SDL_WindowData *window = seat->pointer.pending_frame.enter_window;
+    SDL_WindowData *window = Wayland_GetWindowDataForOwnedSurface(seat->pointer.pending_frame.enter_surface);
+    if (!window) {
+        // Entering a surface not managed by SDL; just set the cursor reset flag.
+        Wayland_SeatResetCursor(seat);
+        return;
+    }
+
+    if (window->surface != seat->pointer.pending_frame.enter_surface) {
+        /* This surface is part of the window managed by SDL, but it is not the main content
+         * surface and doesn't get focus. Just set the default cursor and leave.
+         */
+        Wayland_SeatSetDefaultCursor(seat);
+        return;
+    }
 
     seat->pointer.focus = window;
     ++window->pointer_focus_count;
@@ -834,14 +847,8 @@ static void pointer_handle_enter(void *data, struct wl_pointer *pointer,
         return;
     }
 
-    SDL_WindowData *window = Wayland_GetWindowDataForOwnedSurface(surface);
-    if (!window) {
-        // Not a surface owned by SDL.
-        return;
-    }
-
     SDL_WaylandSeat *seat = (SDL_WaylandSeat *)data;
-    seat->pointer.pending_frame.enter_window = window;
+    seat->pointer.pending_frame.enter_surface = surface;
     seat->pointer.enter_serial = serial;
 
     /* In the case of e.g. a pointer confine warp, we may receive an enter
@@ -860,9 +867,11 @@ static void pointer_handle_enter(void *data, struct wl_pointer *pointer,
 
 static void pointer_dispatch_leave(SDL_WaylandSeat *seat, bool update_pointer)
 {
-    SDL_WindowData *window = seat->pointer.pending_frame.leave_window;
+    SDL_WindowData *window = Wayland_GetWindowDataForOwnedSurface(seat->pointer.pending_frame.leave_surface);
 
     if (window) {
+        if (seat->pointer.focus) {
+            if (seat->pointer.focus->surface == seat->pointer.pending_frame.leave_surface) {
                 // Clear the capture flag and raise all buttons
                 window->sdlwindow->flags &= ~SDL_WINDOW_MOUSE_CAPTURE;
 
@@ -888,6 +897,11 @@ static void pointer_dispatch_leave(SDL_WaylandSeat *seat, bool update_pointer)
                     Wayland_SeatUpdatePointerCursor(seat);
                 }
             }
+        } else if (update_pointer) {
+            // Leaving a non-content surface managed by SDL; just set the cursor reset flag.
+            Wayland_SeatResetCursor(seat);
+        }
+    }
 }
 
 static void pointer_handle_leave(void *data, struct wl_pointer *pointer,
@@ -898,15 +912,9 @@ static void pointer_handle_leave(void *data, struct wl_pointer *pointer,
         return;
     }
 
-    SDL_WindowData *window = Wayland_GetWindowDataForOwnedSurface(surface);
-    if (!window) {
-        // Not a surface owned by SDL.
-        return;
-    }
-
     SDL_WaylandSeat *seat = (SDL_WaylandSeat *)data;
-    seat->pointer.pending_frame.leave_window = window;
+    seat->pointer.pending_frame.leave_surface = surface;
-    if (wl_pointer_get_version(seat->pointer.wl_pointer) < WL_POINTER_FRAME_SINCE_VERSION && window == seat->pointer.focus) {
+    if (wl_pointer_get_version(seat->pointer.wl_pointer) < WL_POINTER_FRAME_SINCE_VERSION) {
         pointer_dispatch_leave(seat, true);
     }
 }
@@ -1277,11 +1285,13 @@ static void pointer_handle_frame(void *data, struct wl_pointer *pointer)
 {
     SDL_WaylandSeat *seat = data;
 
-    if (seat->pointer.pending_frame.enter_window) {
+    if (seat->pointer.pending_frame.enter_surface) {
-        if (seat->pointer.focus && seat->pointer.pending_frame.leave_window == seat->pointer.focus) {
+        if (seat->pointer.pending_frame.leave_surface) {
             // Leaving the previous surface before entering a new surface.
             pointer_dispatch_leave(seat, false);
+            seat->pointer.pending_frame.leave_surface = NULL;
         }
+
         pointer_dispatch_enter(seat);
     }
 
@@ -1309,7 +1319,7 @@ static void pointer_handle_frame(void *data, struct wl_pointer *pointer)
         pointer_dispatch_axis(seat);
     }
 
-    if (seat->pointer.focus && seat->pointer.pending_frame.leave_window == seat->pointer.focus) {
+    if (seat->pointer.pending_frame.leave_surface) {
         pointer_dispatch_leave(seat, true);
     }
 
@@ -1435,7 +1445,7 @@ static void touch_handler_down(void *data, struct wl_touch *touch, uint32_t seri
     Wayland_UpdateImplicitGrabSerial(seat, serial);
     window_data = Wayland_GetWindowDataForOwnedSurface(surface);
 
-    if (window_data) {
+    if (window_data && window_data->surface == surface) {
         float x, y;
 
         if (window_data->current.logical_width <= 1) {
@@ -1457,8 +1467,7 @@ static void touch_handler_down(void *data, struct wl_touch *touch, uint32_t seri
     }
 }
 
-static void touch_handler_up(void *data, struct wl_touch *touch, uint32_t serial,
+static void touch_handler_up(void *data, struct wl_touch *touch, uint32_t serial, uint32_t timestamp, int id)
-                             uint32_t timestamp, int id)
 {
     SDL_WaylandSeat *seat = (SDL_WaylandSeat *)data;
     wl_fixed_t fx = 0, fy = 0;
@@ -1469,7 +1478,7 @@ static void touch_handler_up(void *data, struct wl_touch *touch, uint32_t serial
     if (surface) {
         SDL_WindowData *window_data = Wayland_GetWindowDataForOwnedSurface(surface);
 
-        if (window_data) {
+        if (window_data && window_data->surface == surface) {
             const float x = (float)wl_fixed_to_double(fx) / window_data->current.logical_width;
             const float y = (float)wl_fixed_to_double(fy) / window_data->current.logical_height;
 
@@ -1489,8 +1498,7 @@ static void touch_handler_up(void *data, struct wl_touch *touch, uint32_t serial
     }
 }
 
-static void touch_handler_motion(void *data, struct wl_touch *touch, uint32_t timestamp,
+static void touch_handler_motion(void *data, struct wl_touch *touch, uint32_t timestamp, int id, wl_fixed_t fx, wl_fixed_t fy)
-                                 int id, wl_fixed_t fx, wl_fixed_t fy)
 {
     SDL_WaylandSeat *seat = (SDL_WaylandSeat *)data;
     struct wl_surface *surface = NULL;
@@ -1500,7 +1508,7 @@ static void touch_handler_motion(void *data, struct wl_touch *touch, uint32_t ti
     if (surface) {
         SDL_WindowData *window_data = Wayland_GetWindowDataForOwnedSurface(surface);
 
-        if (window_data) {
+        if (window_data && window_data->surface == surface) {
             const float x = (float)wl_fixed_to_double(fx) / window_data->current.logical_width;
             const float y = (float)wl_fixed_to_double(fy) / window_data->current.logical_height;
 
@@ -2395,9 +2403,9 @@ static void Wayland_SeatDestroyPointer(SDL_WaylandSeat *seat)
 
     // Make sure focus is removed from a surface before the pointer is destroyed.
     if (seat->pointer.focus) {
-        seat->pointer.pending_frame.leave_window = seat->pointer.focus;
+        seat->pointer.pending_frame.leave_surface = seat->pointer.focus->surface;
         pointer_dispatch_leave(seat, false);
-        seat->pointer.pending_frame.leave_window = NULL;
+        seat->pointer.pending_frame.leave_surface = NULL;
     }
 
     SDL_RemoveMouse(seat->pointer.sdl_id);
@@ -3349,7 +3357,7 @@ static void tablet_tool_handle_proximity_in(void *data, struct zwp_tablet_tool_v
 {
     SDL_WaylandPenTool *sdltool = (SDL_WaylandPenTool *) data;
     SDL_WindowData *windowdata = surface ? Wayland_GetWindowDataForOwnedSurface(surface) : NULL;
-    sdltool->focus = windowdata;
+    sdltool->focus = windowdata && windowdata->surface == surface ? windowdata : NULL;
     sdltool->proximity_serial = serial;
     sdltool->frame.have_proximity = true;
     sdltool->frame.in_proximity = true;
@@ -3664,9 +3672,9 @@ void Wayland_DisplayRemoveWindowReferencesFromSeats(SDL_VideoData *display, SDL_
         }
 
         if (seat->pointer.focus == window) {
-            seat->pointer.pending_frame.leave_window = seat->pointer.focus;
+            seat->pointer.pending_frame.leave_surface = seat->pointer.focus->surface;
             pointer_dispatch_leave(seat, true);
-            seat->pointer.pending_frame.leave_window = NULL;
+            seat->pointer.pending_frame.leave_surface = NULL;
         }
 
         // Need the safe loop variant here as cancelling a touch point removes it from the list.

src/video/wayland/SDL_waylandevents_c.h

@@ -236,8 +236,8 @@ typedef struct SDL_WaylandSeat
                 SDL_MouseWheelDirection direction;
             } axis;
 
-            SDL_WindowData *enter_window;
+            struct wl_surface *enter_surface;
-            SDL_WindowData *leave_window;
+            struct wl_surface *leave_surface;
 
             // Event timestamp in nanoseconds
             Uint64 timestamp_ns;

src/video/wayland/SDL_waylandmouse.c

@@ -1523,6 +1523,23 @@ void Wayland_FiniMouse(SDL_VideoData *data)
 #endif
 }
 
+void Wayland_SeatResetCursor(SDL_WaylandSeat *seat)
+{
+    Wayland_CursorStateResetCursor(&seat->pointer.cursor_state);
+}
+
+void Wayland_SeatSetDefaultCursor(SDL_WaylandSeat *seat)
+{
+    SDL_Mouse *mouse = SDL_GetMouse();
+    SDL_WindowData *pointer_focus = seat->pointer.focus;
+    const Wayland_PointerObject obj = {
+        .wl_pointer = seat->pointer.wl_pointer,
+        .is_pointer = true
+    };
+
+    Wayland_CursorStateSetCursor(&seat->pointer.cursor_state, &obj, pointer_focus, seat->pointer.enter_serial, mouse->def_cursor);
+}
+
 void Wayland_SeatUpdatePointerCursor(SDL_WaylandSeat *seat)
 {
     SDL_Mouse *mouse = SDL_GetMouse();

src/video/wayland/SDL_waylandmouse.h

@@ -27,6 +27,8 @@
 extern void Wayland_InitMouse(SDL_VideoData *data);
 extern void Wayland_FiniMouse(SDL_VideoData *data);
 extern void Wayland_SeatUpdatePointerCursor(SDL_WaylandSeat *seat);
+extern void Wayland_SeatSetDefaultCursor(SDL_WaylandSeat *seat);
+extern void Wayland_SeatResetCursor(SDL_WaylandSeat *seat);
 extern void Wayland_TabletToolUpdateCursor(SDL_WaylandPenTool *tool);
 extern void Wayland_SeatWarpMouse(SDL_WaylandSeat *seat, SDL_WindowData *window, float x, float y);
 extern void Wayland_CursorStateSetFrameCallback(SDL_WaylandCursorState *state, void *userdata);

src/video/wayland/SDL_waylandshmbuffer.c

@@ -32,6 +32,7 @@
 
 #include "SDL_waylandshmbuffer.h"
 #include "SDL_waylandvideo.h"
+#include "single-pixel-buffer-v1-client-protocol.h"
 
 static bool SetTempFileSize(int fd, off_t size)
 {
@@ -186,4 +187,28 @@ void Wayland_ReleaseSHMPool(Wayland_SHMPool *shmPool)
     }
 }
 
+struct wl_buffer *Wayland_CreateSinglePixelBuffer(Uint32 r, Uint32 g, Uint32 b, Uint32 a)
+{
+    SDL_VideoData *viddata = SDL_GetVideoDevice()->internal;
+
+    // The single-pixel buffer protocol is preferred, as the compositor can choose an optimal format.
+    if (viddata->single_pixel_buffer_manager) {
+        return wp_single_pixel_buffer_manager_v1_create_u32_rgba_buffer(viddata->single_pixel_buffer_manager, r, g, b, a);
+    } else {
+        Wayland_SHMPool *pool = Wayland_AllocSHMPool(4);
+        if (!pool) {
+            return NULL;
+        }
+
+        void *mem;
+        struct wl_buffer *wl_buffer = Wayland_AllocBufferFromPool(pool, 1, 1, &mem);
+
+        const Uint8 pixel[4] = { r >> 24, g >> 24, b >> 24, a >> 24 };
+        SDL_memcpy(mem, pixel, sizeof(pixel));
+
+        Wayland_ReleaseSHMPool(pool);
+        return wl_buffer;
+    }
+}
+
 #endif

src/video/wayland/SDL_waylandshmbuffer.h

@@ -30,4 +30,6 @@ extern Wayland_SHMPool *Wayland_AllocSHMPool(int size);
 extern struct wl_buffer *Wayland_AllocBufferFromPool(Wayland_SHMPool *shmPool, int width, int height, void **data);
 extern void Wayland_ReleaseSHMPool(Wayland_SHMPool *shmPool);
 
+extern struct wl_buffer *Wayland_CreateSinglePixelBuffer(Uint32 r, Uint32 g, Uint32 b, Uint32 a);
+
 #endif

src/video/wayland/SDL_waylandvideo.c

@@ -69,6 +69,7 @@
 #include "color-management-v1-client-protocol.h"
 #include "pointer-warp-v1-client-protocol.h"
 #include "pointer-gestures-unstable-v1-client-protocol.h"
+#include "single-pixel-buffer-v1-client-protocol.h"
 
 #ifdef HAVE_LIBDECOR_H
 #include <libdecor.h>
@@ -653,6 +654,7 @@ static SDL_VideoDevice *Wayland_CreateDevice(bool require_preferred_protocols)
     device->SetWindowResizable = Wayland_SetWindowResizable;
     device->SetWindowPosition = Wayland_SetWindowPosition;
     device->SetWindowSize = Wayland_SetWindowSize;
+    device->SetWindowAspectRatio = Wayland_SetWindowAspectRatio;
     device->SetWindowMinimumSize = Wayland_SetWindowMinimumSize;
     device->SetWindowMaximumSize = Wayland_SetWindowMaximumSize;
     device->SetWindowParent = Wayland_SetWindowParent;
@@ -1278,6 +1280,8 @@ static void handle_registry_global(void *data, struct wl_registry *registry, uin
 
     if (SDL_strcmp(interface, "wl_compositor") == 0) {
         d->compositor = wl_registry_bind(d->registry, id, &wl_compositor_interface, SDL_min(SDL_WL_COMPOSITOR_VERSION, version));
+    } else if (SDL_strcmp(interface, "wl_subcompositor") == 0) {
+        d->subcompositor = wl_registry_bind(d->registry, id, &wl_subcompositor_interface, 1);
     } else if (SDL_strcmp(interface, "wl_output") == 0) {
         Wayland_add_display(d, id, SDL_min(version, SDL_WL_OUTPUT_VERSION));
     } else if (SDL_strcmp(interface, "wl_seat") == 0) {
@@ -1344,6 +1348,8 @@ static void handle_registry_global(void *data, struct wl_registry *registry, uin
     } else if (SDL_strcmp(interface, "zwp_pointer_gestures_v1") == 0) {
         d->zwp_pointer_gestures = wl_registry_bind(d->registry, id, &zwp_pointer_gestures_v1_interface, SDL_min(version, 3));
         Wayland_DisplayInitPointerGestureManager(d);
+    } else if (SDL_strcmp(interface, "wp_single_pixel_buffer_manager_v1") == 0) {
+        d->single_pixel_buffer_manager = wl_registry_bind(d->registry, id, &wp_single_pixel_buffer_manager_v1_interface, 1);
     }
 #ifdef SDL_WL_FIXES_VERSION
     else if (SDL_strcmp(interface, "wl_fixes") == 0) {
@@ -1692,6 +1698,16 @@ static void Wayland_VideoCleanup(SDL_VideoDevice *_this)
         data->zwp_pointer_gestures = NULL;
     }
 
+    if (data->single_pixel_buffer_manager) {
+        wp_single_pixel_buffer_manager_v1_destroy(data->single_pixel_buffer_manager);
+        data->single_pixel_buffer_manager = NULL;
+    }
+
+    if (data->subcompositor) {
+        wl_subcompositor_destroy(data->subcompositor);
+        data->subcompositor = NULL;
+    }
+
     if (data->compositor) {
         wl_compositor_destroy(data->compositor);
         data->compositor = NULL;

src/video/wayland/SDL_waylandvideo.h

@@ -61,6 +61,7 @@ struct SDL_VideoData
         struct libdecor *libdecor;
 #endif
     } shell;
+    struct wl_subcompositor *subcompositor;
     struct zwp_relative_pointer_manager_v1 *relative_pointer_manager;
     struct zwp_pointer_constraints_v1 *pointer_constraints;
     struct wp_pointer_warp_v1 *wp_pointer_warp_v1;
@@ -85,6 +86,7 @@ struct SDL_VideoData
     struct zwp_tablet_manager_v2 *tablet_manager;
     struct wl_fixes *wl_fixes;
     struct zwp_pointer_gestures_v1 *zwp_pointer_gestures;
+    struct wp_single_pixel_buffer_manager_v1 *single_pixel_buffer_manager;
 
     struct xkb_context *xkb_context;
 

src/video/wayland/SDL_waylandwindow.h

@@ -165,6 +165,10 @@ struct SDL_WindowData
         // The size of the window backbuffer in pixels.
         int pixel_width;
         int pixel_height;
+
+        // The dimensions of the active viewport, in logical units.
+        int viewport_width;
+        int viewport_height;
     } current;
 
     // The last compositor requested parameters; used for deduplication of window geometry configuration.
@@ -188,6 +192,20 @@ struct SDL_WindowData
         int height;
     } toplevel_bounds;
 
+    struct
+    {
+        struct wl_surface *surface;
+        struct wl_subsurface *subsurface;
+        struct wl_buffer *buffer;
+        struct wp_viewport *viewport;
+
+        int offset_x;
+        int offset_y;
+
+        bool mapped;
+        bool opaque;
+    } mask;
+
     struct
     {
         int hint;
@@ -196,8 +214,7 @@ struct SDL_WindowData
     } text_input_props;
 
     SDL_DisplayID last_displayID;
-    int fullscreen_deadline_count;
+    int pending_state_deadline_count;
-    int maximized_restored_deadline_count;
     Uint64 last_focus_event_time_ns;
     int icc_fd;
     Uint32 icc_size;
@@ -206,6 +223,8 @@ struct SDL_WindowData
     bool resizing;
     bool active;
     bool pending_config_ack;
+    bool pending_state_commit;
+    bool limits_changed;
     bool is_fullscreen;
     bool fullscreen_exclusive;
     bool drop_fullscreen_requests;
@@ -236,6 +255,7 @@ extern void Wayland_SetWindowResizable(SDL_VideoDevice *_this, SDL_Window *windo
 extern bool Wayland_CreateWindow(SDL_VideoDevice *_this, SDL_Window *window, SDL_PropertiesID create_props);
 extern bool Wayland_SetWindowPosition(SDL_VideoDevice *_this, SDL_Window *window);
 extern void Wayland_SetWindowSize(SDL_VideoDevice *_this, SDL_Window *window);
+extern void Wayland_SetWindowAspectRatio(SDL_VideoDevice *_this, SDL_Window *window);
 extern void Wayland_SetWindowMinimumSize(SDL_VideoDevice *_this, SDL_Window *window);
 extern void Wayland_SetWindowMaximumSize(SDL_VideoDevice *_this, SDL_Window *window);
 extern void Wayland_GetWindowSizeInPixels(SDL_VideoDevice *_this, SDL_Window *window, int *w, int *h);

wayland-protocols/single-pixel-buffer-v1.xml

@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<protocol name="single_pixel_buffer_v1">
+  <copyright>
+    Copyright © 2022 Simon Ser
+
+    Permission is hereby granted, free of charge, to any person obtaining a
+    copy of this software and associated documentation files (the "Software"),
+    to deal in the Software without restriction, including without limitation
+    the rights to use, copy, modify, merge, publish, distribute, sublicense,
+    and/or sell copies of the Software, and to permit persons to whom the
+    Software is furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice (including the next
+    paragraph) shall be included in all copies or substantial portions of the
+    Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+    THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+    DEALINGS IN THE SOFTWARE.
+  </copyright>
+
+  <description summary="single pixel buffer factory">
+    This protocol extension allows clients to create single-pixel buffers.
+
+    Compositors supporting this protocol extension should also support the
+    viewporter protocol extension. Clients may use viewporter to scale a
+    single-pixel buffer to a desired size.
+
+    Warning! The protocol described in this file is currently in the testing
+    phase. Backward compatible changes may be added together with the
+    corresponding interface version bump. Backward incompatible changes can
+    only be done by creating a new major version of the extension.
+  </description>
+
+  <interface name="wp_single_pixel_buffer_manager_v1" version="1">
+    <description summary="global factory for single-pixel buffers">
+      The wp_single_pixel_buffer_manager_v1 interface is a factory for
+      single-pixel buffers.
+    </description>
+
+    <request name="destroy" type="destructor">
+      <description summary="destroy the manager">
+        Destroy the wp_single_pixel_buffer_manager_v1 object.
+
+        The child objects created via this interface are unaffected.
+      </description>
+    </request>
+
+    <request name="create_u32_rgba_buffer">
+      <description summary="create a 1×1 buffer from 32-bit RGBA values">
+        Create a single-pixel buffer from four 32-bit RGBA values.
+
+        Unless specified in another protocol extension, the RGBA values use
+        pre-multiplied alpha.
+
+        The width and height of the buffer are 1.
+
+        The r, g, b and a arguments valid range is from UINT32_MIN (0)
+        to UINT32_MAX (0xffffffff).
+        
+        These arguments should be interpreted as a percentage, i.e.
+        - UINT32_MIN = 0% of the given color component
+        - UINT32_MAX = 100% of the given color component
+      </description>
+      <arg name="id" type="new_id" interface="wl_buffer"/>
+      <arg name="r" type="uint" summary="value of the buffer's red channel"/>
+      <arg name="g" type="uint" summary="value of the buffer's green channel"/>
+      <arg name="b" type="uint" summary="value of the buffer's blue channel"/>
+      <arg name="a" type="uint" summary="value of the buffer's alpha channel"/>
+    </request>
+  </interface>
+</protocol>