shape: More robust handling of failure cases in CreateShaper.

src/video/cocoa/SDL_cocoashape.m

@@ -47,11 +47,17 @@ Cocoa_CreateShaper(SDL_Window* window)
     SDL_ShapeData* data;
     int resized_properly;
     SDL_WindowData* windata = (__bridge SDL_WindowData*)window->driverdata;
+
+    result = (SDL_WindowShaper *)SDL_malloc(sizeof(SDL_WindowShaper));
+    if (!result) {
+        SDL_OutOfMemory();
+        return NULL;
+    }
+
     [windata.nswindow setOpaque:NO];
 
     [windata.nswindow setStyleMask:NSWindowStyleMaskBorderless];
 
-    result = (SDL_WindowShaper *)SDL_malloc(sizeof(SDL_WindowShaper));
     result->window = window;
     result->mode.mode = ShapeModeDefault;
     result->mode.parameters.binarizationCutoff = 1;

src/video/directfb/SDL_DirectFB_shape.c

@@ -35,11 +35,20 @@ DirectFB_CreateShaper(SDL_Window* window) {
     int resized_properly;
 
     result = SDL_malloc(sizeof(SDL_WindowShaper));
+    if (!result) {
+        SDL_OutOfMemory();
+        return NULL;
+    }
     result->window = window;
     result->mode.mode = ShapeModeDefault;
     result->mode.parameters.binarizationCutoff = 1;
     result->userx = result->usery = 0;
     data = SDL_malloc(sizeof(SDL_ShapeData));
+    if (!data) {
+        SDL_free(result);
+        SDL_OutOfMemory();
+        return NULL;
+    }
     result->driverdata = data;
     data->surface = NULL;
     window->shaper = result;

src/video/windows/SDL_windowsshape.c

@@ -29,13 +29,21 @@ SDL_WindowShaper*
 Win32_CreateShaper(SDL_Window * window) {
     int resized_properly;
     SDL_WindowShaper* result = (SDL_WindowShaper *)SDL_malloc(sizeof(SDL_WindowShaper));
+    if (!result) {
+        SDL_OutOfMemory();
+        return NULL;
+    }
     result->window = window;
     result->mode.mode = ShapeModeDefault;
     result->mode.parameters.binarizationCutoff = 1;
     result->userx = result->usery = 0;
     result->hasshape = SDL_FALSE;
-    result->driverdata = (SDL_ShapeData*)SDL_malloc(sizeof(SDL_ShapeData));
+    result->driverdata = (SDL_ShapeData*)SDL_calloc(1, sizeof(SDL_ShapeData));
-    ((SDL_ShapeData*)result->driverdata)->mask_tree = NULL;
+    if (!result->driverdata) {
+        SDL_free(result);
+        SDL_OutOfMemory();
+        return NULL;
+    }
     window->shaper = result;
     /* Put some driver-data here. */
     resized_properly = Win32_ResizeWindowShape(window);

src/video/x11/SDL_x11shape.c

@@ -31,22 +31,34 @@ SDL_WindowShaper*
 X11_CreateShaper(SDL_Window* window) {
     SDL_WindowShaper* result = NULL;
     SDL_ShapeData* data = NULL;
-    int resized_properly;
 
 #if SDL_VIDEO_DRIVER_X11_XSHAPE
     if (SDL_X11_HAVE_XSHAPE) {  /* Make sure X server supports it. */
         result = SDL_malloc(sizeof(SDL_WindowShaper));
+        if (!result) {
+            SDL_OutOfMemory();
+            return NULL;
+        }
         result->window = window;
         result->mode.mode = ShapeModeDefault;
         result->mode.parameters.binarizationCutoff = 1;
         result->userx = result->usery = 0;
         data = SDL_malloc(sizeof(SDL_ShapeData));
+        if (!data) {
+            SDL_free(result);
+            SDL_OutOfMemory();
+            return NULL;
+        }
         result->driverdata = data;
         data->bitmapsize = 0;
         data->bitmap = NULL;
         window->shaper = result;
-        resized_properly = X11_ResizeWindowShape(window);
+        if (X11_ResizeWindowShape(window) != 0) {
-        SDL_assert(resized_properly == 0);
+            SDL_free(result);
+            SDL_free(data);
+            window->shaper = NULL;
+            return NULL;
+        }
     }
 #endif
 
@@ -64,11 +76,10 @@ X11_ResizeWindowShape(SDL_Window* window) {
     bitmapsize *= window->h;
     if(data->bitmapsize != bitmapsize || data->bitmap == NULL) {
         data->bitmapsize = bitmapsize;
-        if(data->bitmap != NULL)
+        SDL_free(data->bitmap);
-            SDL_free(data->bitmap);
         data->bitmap = SDL_malloc(data->bitmapsize);
         if(data->bitmap == NULL) {
-            return SDL_SetError("Could not allocate memory for shaped-window bitmap.");
+            return SDL_OutOfMemory();
         }
     }
     SDL_memset(data->bitmap,0,data->bitmapsize);