diff --git a/src/video/x11/SDL_x11xinput2.c b/src/video/x11/SDL_x11xinput2.c
index 05b6c6e3e9..7c8f604a90 100644
--- a/src/video/x11/SDL_x11xinput2.c
+++ b/src/video/x11/SDL_x11xinput2.c
@@ -590,6 +590,9 @@ void X11_HandleXinput2Event(SDL_VideoDevice *_this, XGenericEventCookie *cookie)
         bool pointer_emulated = false;
 #endif
 
+        // Store the button serial to filter out redundant core button events.
+        videodata->xinput_last_button_serial = xev->serial;
+
         if (pen) {
             if (xev->deviceid != xev->sourceid) {
                 // Discard events from "Master" devices to avoid duplicates.
@@ -607,9 +610,6 @@ void X11_HandleXinput2Event(SDL_VideoDevice *_this, XGenericEventCookie *cookie)
             SDL_WindowData *windowdata = X11_FindWindow(videodata, xev->event);
             int x_ticks = 0, y_ticks = 0;
 
-            // Store the button serial to filter out redundant core button events.
-            videodata->xinput_last_button_serial = xev->serial;
-
             if (xev->deviceid != videodata->xinput_master_pointer_device) {
                 // Ignore slave button events on non-focused windows, or focus can be incorrectly set while a grab is active.
                 if (SDL_GetMouseFocus() != windowdata->window) {