encounter
/
SDL
mirror of https://github.com/encounter/SDL.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

video: Harden calculation of SDL_surface pitch and size against overflow 

If the width is sufficiently ludicrous, then the calculated pitch or
the image size could conceivably be a signed integer overflow, which
is undefined behaviour. Calculate in the unsigned size_t domain, with
overflow checks.

Signed-off-by: Simon McVittie <smcv@collabora.com>
This commit is contained in:
Simon McVittie 2022-05-10 10:33:54 +01:00 committed by Sam Lantinga
parent 6606ab8b56
commit 8c9f7104e3
1 changed files with 28 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
src/video/SDL_surface.c
View File

@ -33,22 +33,37 @@
SDL_COMPILE_TIME_ASSERT(surface_size_assumptions, SDL_COMPILE_TIME_ASSERT(surface_size_assumptions,
sizeof(int) == sizeof(Sint32) && sizeof(size_t) >= sizeof(Sint32)); sizeof(int) == sizeof(Sint32) && sizeof(size_t) >= sizeof(Sint32));
SDL_COMPILE_TIME_ASSERT(can_indicate_overflow, SDL_SIZE_MAX > SDL_MAX_SINT32);
/* Public routines */ /* Public routines */
/* /*
* Calculate the pad-aligned scanline width of a surface * Calculate the pad-aligned scanline width of a surface.
* Return SDL_SIZE_MAX on overflow.
*/ */
static Sint64 static size_t
SDL_CalculatePitch(Uint32 format, int width) SDL_CalculatePitch(Uint32 format, size_t width)
{ {
Sint64 pitch; size_t pitch;
if (SDL_ISPIXELFORMAT_FOURCC(format) || SDL_BITSPERPIXEL(format) >= 8) { if (SDL_ISPIXELFORMAT_FOURCC(format) || SDL_BITSPERPIXEL(format) >= 8) {
pitch = ((Sint64)width * SDL_BYTESPERPIXEL(format)); if (SDL_size_mul_overflow(width, SDL_BYTESPERPIXEL(format), &pitch)) {
return SDL_SIZE_MAX;
}
} else { } else {
pitch = (((Sint64)width * SDL_BITSPERPIXEL(format)) + 7) / 8; if (SDL_size_mul_overflow(width, SDL_BITSPERPIXEL(format), &pitch)) {
return SDL_SIZE_MAX;
}
if (SDL_size_add_overflow(pitch, 7, &pitch)) {
return SDL_SIZE_MAX;
}
pitch /= 8;
} }
pitch = (pitch + 3) & ~3; /* 4-byte aligning for speed */ /* 4-byte aligning for speed */
if (SDL_size_add_overflow(pitch, 3, &pitch)) {
return SDL_SIZE_MAX;
}
pitch &= ~3;
return pitch; return pitch;
} }
@ -60,14 +75,14 @@ SDL_Surface *
SDL_CreateRGBSurfaceWithFormat(Uint32 flags, int width, int height, int depth, SDL_CreateRGBSurfaceWithFormat(Uint32 flags, int width, int height, int depth,
Uint32 format) Uint32 format)
{ {
Sint64 pitch; size_t pitch;
SDL_Surface *surface; SDL_Surface *surface;
/* The flags are no longer used, make the compiler happy */ /* The flags are no longer used, make the compiler happy */
(void)flags; (void)flags;
pitch = SDL_CalculatePitch(format, width); pitch = SDL_CalculatePitch(format, width);
if (pitch < 0 || pitch > SDL_MAX_SINT32) { if (pitch > SDL_MAX_SINT32) {
/* Overflow... */ /* Overflow... */
SDL_OutOfMemory(); SDL_OutOfMemory();
return NULL; return NULL;
@ -113,15 +128,15 @@ SDL_CreateRGBSurfaceWithFormat(Uint32 flags, int width, int height, int depth,
/* Get the pixels */ /* Get the pixels */
if (surface->w && surface->h) { if (surface->w && surface->h) {
/* Assumptions checked in surface_size_assumptions assert above */ /* Assumptions checked in surface_size_assumptions assert above */
Sint64 size = ((Sint64)surface->h * surface->pitch); size_t size;
if (size < 0 || size > SDL_MAX_SINT32) { if (SDL_size_mul_overflow(surface->h, surface->pitch, &size)) {
/* Overflow... */ /* Overflow... */
SDL_FreeSurface(surface); SDL_FreeSurface(surface);
SDL_OutOfMemory(); SDL_OutOfMemory();
return NULL; return NULL;
} }
surface->pixels = SDL_SIMDAlloc((size_t)size); surface->pixels = SDL_SIMDAlloc(size);
if (!surface->pixels) { if (!surface->pixels) {
SDL_FreeSurface(surface); SDL_FreeSurface(surface);
SDL_OutOfMemory(); SDL_OutOfMemory();
@ -129,7 +144,7 @@ SDL_CreateRGBSurfaceWithFormat(Uint32 flags, int width, int height, int depth,
} }
surface->flags |= SDL_SIMD_ALIGNED; surface->flags |= SDL_SIMD_ALIGNED;
/* This is important for bitmaps */ /* This is important for bitmaps */
SDL_memset(surface->pixels, 0, surface->h * surface->pitch); SDL_memset(surface->pixels, 0, size);
} }
/* Allocate an empty mapping */ /* Allocate an empty mapping */