DevKit7/libctru
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed buffer overflow after gfxSetScreenFormat (#318)

Browse Source
This commit is contained in:
Matthias Lutter 2016-11-10 23:24:23 +01:00 committed by fincs
parent f7eed42835
commit 9048408bd3
1 changed files with 55 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

82
libctru/source/gfx.c
View File

@ -16,6 +16,8 @@ u8* gfxSharedMemory;
u8* gfxTopLeftFramebuffers[2]; u8* gfxTopLeftFramebuffers[2];
u8* gfxTopRightFramebuffers[2]; u8* gfxTopRightFramebuffers[2];
u8* gfxBottomFramebuffers[2]; u8* gfxBottomFramebuffers[2];
u32 gfxTopFramebufferMaxSize;
u32 gfxBottomFramebufferMaxSize;
static bool enable3d; static bool enable3d;
static u8 currentBuffer[2]; static u8 currentBuffer[2];
@ -28,6 +30,9 @@ static GSPGPU_FramebufferFormats botFormat = GSP_BGR8_OES;
static GSPGPU_FramebufferInfo* const framebufferInfoSt[] = { &topFramebufferInfo, &bottomFramebufferInfo }; static GSPGPU_FramebufferInfo* const framebufferInfoSt[] = { &topFramebufferInfo, &bottomFramebufferInfo };
static void (*screenFree)(void *) = NULL;
static void *(*screenAlloc)(size_t) = NULL;
void gfxSet3D(bool enable) void gfxSet3D(bool enable)
{ {
enable3d=enable; enable3d=enable;
@ -38,39 +43,64 @@ bool gfxIs3D(void)
return enable3d; return enable3d;
} }
u32 __get_bytes_per_pixel(GSPGPU_FramebufferFormats format) {
switch(format) {
case GSP_RGBA8_OES:
return 4;
case GSP_BGR8_OES:
return 3;
case GSP_RGB565_OES:
case GSP_RGB5_A1_OES:
case GSP_RGBA4_OES:
return 2;
}
return 3;
}
void gfxSetScreenFormat(gfxScreen_t screen, GSPGPU_FramebufferFormats format) { void gfxSetScreenFormat(gfxScreen_t screen, GSPGPU_FramebufferFormats format) {
if(screen==GFX_TOP) if (screenAlloc == NULL || screenFree == NULL) return;
topFormat = format; if(screen==GFX_TOP)
else {
botFormat = format; u32 topSize = 400 * 240 * __get_bytes_per_pixel(format);
if (gfxTopFramebufferMaxSize < topSize)
{
screenFree(gfxTopLeftFramebuffers[0]);
screenFree(gfxTopLeftFramebuffers[1]);
screenFree(gfxTopRightFramebuffers[0]);
screenFree(gfxTopRightFramebuffers[1]);
gfxTopLeftFramebuffers[0]=screenAlloc(topSize);
gfxTopLeftFramebuffers[1]=screenAlloc(topSize);
gfxTopRightFramebuffers[0]=screenAlloc(topSize);
gfxTopRightFramebuffers[1]=screenAlloc(topSize);
gfxTopFramebufferMaxSize = topSize;
}
topFormat = format;
}else{
u32 bottomSize = 320 * 240 * __get_bytes_per_pixel(format);
if (gfxBottomFramebufferMaxSize < bottomSize)
{
screenFree(gfxBottomFramebuffers[0]);
screenFree(gfxBottomFramebuffers[1]);
gfxBottomFramebuffers[0]=screenAlloc(bottomSize);
gfxBottomFramebuffers[1]=screenAlloc(bottomSize);
gfxBottomFramebufferMaxSize = bottomSize;
}
botFormat = format;
}
} }
GSPGPU_FramebufferFormats gfxGetScreenFormat(gfxScreen_t screen) { GSPGPU_FramebufferFormats gfxGetScreenFormat(gfxScreen_t screen) {
if(screen==GFX_TOP) if(screen==GFX_TOP)
return topFormat; return topFormat;
else else
return botFormat; return botFormat;
} }
void gfxSetDoubleBuffering(gfxScreen_t screen, bool doubleBuffering) { void gfxSetDoubleBuffering(gfxScreen_t screen, bool doubleBuffering) {
doubleBuf[screen] = doubleBuffering ? 1 : 0; // make sure they're the integer values '1' and '0' doubleBuf[screen] = doubleBuffering ? 1 : 0; // make sure they're the integer values '1' and '0'
} }
u32 __get_bytes_per_pixel(GSPGPU_FramebufferFormats format) {
switch(format) {
case GSP_RGBA8_OES:
return 4;
case GSP_BGR8_OES:
return 3;
case GSP_RGB565_OES:
case GSP_RGB5_A1_OES:
case GSP_RGBA4_OES:
return 2;
}
return 3;
}
void gfxSetFramebufferInfo(gfxScreen_t screen, u8 id) void gfxSetFramebufferInfo(gfxScreen_t screen, u8 id)
{ {
if(screen==GFX_TOP) if(screen==GFX_TOP)
@ -105,12 +135,8 @@ void gfxWriteFramebufferInfo(gfxScreen_t screen)
framebufferInfoHeader[0x1]=1; framebufferInfoHeader[0x1]=1;
} }
static void (*screenFree)(void *) = NULL;
void gfxInit(GSPGPU_FramebufferFormats topFormat, GSPGPU_FramebufferFormats bottomFormat, bool vrambuffers) void gfxInit(GSPGPU_FramebufferFormats topFormat, GSPGPU_FramebufferFormats bottomFormat, bool vrambuffers)
{ {
void *(*screenAlloc)(size_t);
if (vrambuffers) if (vrambuffers)
{ {
screenAlloc=vramAlloc; screenAlloc=vramAlloc;
@ -150,6 +176,8 @@ void gfxInit(GSPGPU_FramebufferFormats topFormat, GSPGPU_FramebufferFormats bott
gfxBottomFramebuffers[1]=screenAlloc(bottomSize); gfxBottomFramebuffers[1]=screenAlloc(bottomSize);
gfxTopRightFramebuffers[0]=screenAlloc(topSize); gfxTopRightFramebuffers[0]=screenAlloc(topSize);
gfxTopRightFramebuffers[1]=screenAlloc(topSize); gfxTopRightFramebuffers[1]=screenAlloc(topSize);
gfxTopFramebufferMaxSize = topSize;
gfxBottomFramebufferMaxSize = bottomSize;
enable3d=false; enable3d=false;