diff --git a/src/render/SDL_yuv_sw.c b/src/render/SDL_yuv_sw.c index f1a37c4f0a..55d02ce00b 100644 --- a/src/render/SDL_yuv_sw.c +++ b/src/render/SDL_yuv_sw.c @@ -58,7 +58,10 @@ SDL_SW_CreateYUVTexture(Uint32 format, int w, int h) swdata->h = h; { size_t dst_size; - SDL_CalculateYUVSize(format, w, h, &dst_size, NULL); + if (SDL_CalculateYUVSize(format, w, h, &dst_size, NULL) < 0) { + SDL_OutOfMemory(); + return NULL; + } swdata->pixels = (Uint8 *)SDL_SIMDAlloc(dst_size); if (!swdata->pixels) { SDL_SW_DestroyYUVTexture(swdata); diff --git a/src/video/SDL_surface.c b/src/video/SDL_surface.c index 0a25af374f..3255b73e7a 100644 --- a/src/video/SDL_surface.c +++ b/src/video/SDL_surface.c @@ -39,13 +39,15 @@ SDL_COMPILE_TIME_ASSERT(can_indicate_overflow, SDL_SIZE_MAX > SDL_MAX_SINT32); /* * Calculate the pad-aligned scanline width of a surface. * Return SDL_SIZE_MAX on overflow. + * + * for FOURCC, use SDL_CalculateYUVSize() */ static size_t SDL_CalculatePitch(Uint32 format, size_t width, SDL_bool minimal) { size_t pitch; - if (SDL_ISPIXELFORMAT_FOURCC(format) || SDL_BITSPERPIXEL(format) >= 8) { + if (SDL_BITSPERPIXEL(format) >= 8) { if (SDL_size_mul_overflow(width, SDL_BYTESPERPIXEL(format), &pitch)) { return SDL_SIZE_MAX; } @@ -88,11 +90,21 @@ SDL_CreateSurface(int width, int height, Uint32 format) return NULL; } - pitch = SDL_CalculatePitch(format, width, SDL_FALSE); - if (pitch > SDL_MAX_SINT32) { - /* Overflow... */ - SDL_OutOfMemory(); - return NULL; + if (SDL_ISPIXELFORMAT_FOURCC(format)) { + int p; + if (SDL_CalculateYUVSize(format, width, height, NULL, &p) < 0) { + /* Overflow... */ + SDL_OutOfMemory(); + return NULL; + } + pitch = p; + } else { + pitch = SDL_CalculatePitch(format, width, SDL_FALSE); + if (pitch > SDL_MAX_SINT32) { + /* Overflow... */ + SDL_OutOfMemory(); + return NULL; + } } /* Allocate the surface */ @@ -134,20 +146,21 @@ SDL_CreateSurface(int width, int height, Uint32 format) /* Get the pixels */ if (surface->w && surface->h) { - /* Assumptions checked in surface_size_assumptions assert above */ size_t size; - if (SDL_size_mul_overflow(surface->h, surface->pitch, &size)) { - /* Overflow... */ - SDL_FreeSurface(surface); - SDL_OutOfMemory(); - return NULL; - } - if (SDL_ISPIXELFORMAT_FOURCC(surface->format->format)) { /* Get correct size and pitch for YUV formats */ if (SDL_CalculateYUVSize(surface->format->format, surface->w, surface->h, &size, &surface->pitch) < 0) { - SDL_OutOfMemory(); + /* Overflow... */ SDL_FreeSurface(surface); + SDL_OutOfMemory(); + return NULL; + } + } else { + /* Assumptions checked in surface_size_assumptions assert above */ + if (SDL_size_mul_overflow(surface->h, surface->pitch, &size)) { + /* Overflow... */ + SDL_FreeSurface(surface); + SDL_OutOfMemory(); return NULL; } } @@ -202,7 +215,16 @@ SDL_CreateSurfaceFrom(void *pixels, return NULL; } - minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE); + if (SDL_ISPIXELFORMAT_FOURCC(format)) { + int p; + if (SDL_CalculateYUVSize(format, width, height, NULL, &p) < 0) { + SDL_InvalidParamError("pitch"); + return NULL; + } + minimalPitch = p; + } else { + minimalPitch = SDL_CalculatePitch(format, width, SDL_TRUE); + } if (pitch < 0 || (pitch > 0 && ((size_t)pitch) < minimalPitch)) { SDL_InvalidParamError("pitch"); diff --git a/src/video/SDL_yuv.c b/src/video/SDL_yuv.c index 773a375daf..aff90926c7 100644 --- a/src/video/SDL_yuv.c +++ b/src/video/SDL_yuv.c @@ -29,6 +29,10 @@ static SDL_YUV_CONVERSION_MODE SDL_YUV_ConversionMode = SDL_YUV_CONVERSION_BT601; +#if SDL_HAVE_YUV +static SDL_bool IsPlanar2x2Format(Uint32 format); +#endif + void SDL_SetYUVConversionMode(SDL_YUV_CONVERSION_MODE mode) { SDL_YUV_ConversionMode = mode; @@ -53,51 +57,130 @@ SDL_YUV_CONVERSION_MODE SDL_GetYUVConversionModeForResolution(int width, int hei } /* - * Calculate YUV size. + * Calculate YUV size and pitch. Check for overflow. * Output 'pitch' that can be used with SDL_ConvertPixels() * * return 0 on success, -1 on error */ int SDL_CalculateYUVSize(Uint32 format, int w, int h, size_t *size, int *pitch) { - const int sz_plane = w * h; - const int sz_plane_chroma = ((w + 1) / 2) * ((h + 1) / 2); - const int sz_plane_packed = ((w + 1) / 2) * h; - int dst_size = 0; +#if SDL_HAVE_YUV + int sz_plane = 0, sz_plane_chroma = 0, sz_plane_packed = 0; + + if (IsPlanar2x2Format(format) == SDL_TRUE) { + { + /* sz_plane == w * h; */ + size_t s1; + if (SDL_size_mul_overflow(w, h, &s1) < 0) { + return -1; + } + sz_plane = (int) s1; + } + + { + /* sz_plane_chroma == ((w + 1) / 2) * ((h + 1) / 2); */ + size_t s1, s2, s3; + if (SDL_size_add_overflow(w, 1, &s1) < 0) { + return -1; + } + s1 = s1 / 2; + if (SDL_size_add_overflow(h, 1, &s2) < 0) { + return -1; + } + s2 = s2 / 2; + if (SDL_size_mul_overflow(s1, s2, &s3) < 0) { + return -1; + } + sz_plane_chroma = (int) s3; + } + } else { + /* sz_plane_packed == ((w + 1) / 2) * h; */ + size_t s1, s2; + if (SDL_size_add_overflow(w, 1, &s1) < 0) { + return -1; + } + s1 = s1 / 2; + if (SDL_size_mul_overflow(s1, h, &s2) < 0) { + return -1; + } + sz_plane_packed = (int) s2; + } + switch (format) { case SDL_PIXELFORMAT_YV12: /**< Planar mode: Y + V + U (3 planes) */ case SDL_PIXELFORMAT_IYUV: /**< Planar mode: Y + U + V (3 planes) */ - dst_size = sz_plane + sz_plane_chroma + sz_plane_chroma; + if (pitch) { *pitch = w; } + + if (size) { + /* dst_size == sz_plane + sz_plane_chroma + sz_plane_chroma; */ + size_t s1, s2; + if (SDL_size_add_overflow(sz_plane, sz_plane_chroma, &s1) < 0) { + return -1; + } + if (SDL_size_add_overflow(s1, sz_plane_chroma, &s2) < 0) { + return -1; + } + *size = (int)s2; + } break; case SDL_PIXELFORMAT_YUY2: /**< Packed mode: Y0+U0+Y1+V0 (1 plane) */ case SDL_PIXELFORMAT_UYVY: /**< Packed mode: U0+Y0+V0+Y1 (1 plane) */ case SDL_PIXELFORMAT_YVYU: /**< Packed mode: Y0+V0+Y1+U0 (1 plane) */ - dst_size = 4 * sz_plane_packed; + if (pitch) { - *pitch = ((w + 1) / 2) * 4; + /* pitch == ((w + 1) / 2) * 4; */ + size_t p1, p2; + if (SDL_size_add_overflow(w, 1, &p1) < 0) { + return -1; + } + p1 = p1 / 2; + if (SDL_size_mul_overflow(p1, 4, &p2) < 0) { + return -1; + } + *pitch = (int) p2; + } + + if (size) { + /* dst_size == 4 * sz_plane_packed; */ + size_t s1; + if (SDL_size_mul_overflow(sz_plane_packed, 4, &s1) < 0) { + return -1; + } + *size = (int) s1; } break; case SDL_PIXELFORMAT_NV12: /**< Planar mode: Y + U/V interleaved (2 planes) */ case SDL_PIXELFORMAT_NV21: /**< Planar mode: Y + V/U interleaved (2 planes) */ - dst_size = sz_plane + sz_plane_chroma + sz_plane_chroma; if (pitch) { *pitch = w; } + + if (size) { + /* dst_size == sz_plane + sz_plane_chroma + sz_plane_chroma; */ + size_t s1, s2; + if (SDL_size_add_overflow(sz_plane, sz_plane_chroma, &s1) < 0) { + return -1; + } + if (SDL_size_add_overflow(s1, sz_plane_chroma, &s2) < 0) { + return -1; + } + *size = (int) s2; + } break; default: return -1; } - if (size) { - *size = dst_size; - } return 0; +#else + return -1; +#endif } #if SDL_HAVE_YUV