Fixed creating a metal renderer without specifying a metal window

src/render/metal/SDL_render_metal.m

@@ -47,6 +47,9 @@
 
 /* Apple Metal renderer implementation */
 
+/* Used to re-create the window with Metal capability */
+extern int SDL_RecreateWindow(SDL_Window * window, Uint32 flags);
+
 /* macOS requires constants in a buffer to have a 256 byte alignment. */
 /* Use native type alignments from https://developer.apple.com/metal/Metal-Shading-Language-Specification.pdf */
 #ifdef __MACOSX__
@@ -1578,6 +1581,8 @@ METAL_CreateRenderer(SDL_Window * window, Uint32 flags)
     SDL_MetalView view = NULL;
     CAMetalLayer *layer = nil;
     SDL_SysWMinfo syswm;
+    Uint32 window_flags;
+    SDL_bool changed_window = SDL_FALSE;
 
     SDL_VERSION(&syswm.version);
     if (!SDL_GetWindowWMInfo(window, &syswm)) {
@@ -1588,10 +1593,18 @@ METAL_CreateRenderer(SDL_Window * window, Uint32 flags)
         return NULL;
     }
 
+    window_flags = SDL_GetWindowFlags(window);
+    if (!(window_flags & SDL_WINDOW_METAL)) {
+        changed_window = SDL_TRUE;
+        if (SDL_RecreateWindow(window, (window_flags & ~SDL_WINDOW_OPENGL) | SDL_WINDOW_METAL) < 0) {
+            goto error;
+        }
+    }
+
     renderer = (SDL_Renderer *) SDL_calloc(1, sizeof(*renderer));
     if (!renderer) {
         SDL_OutOfMemory();
-        return NULL;
+        goto error;
     }
 
     // !!! FIXME: MTLCopyAllDevices() can find other GPUs on macOS...
@@ -1600,7 +1613,7 @@ METAL_CreateRenderer(SDL_Window * window, Uint32 flags)
     if (mtldevice == nil) {
         SDL_free(renderer);
         SDL_SetError("Failed to obtain Metal device");
-        return NULL;
+        goto error;
     }
 
     view = SDL_Metal_CreateView(window);
@@ -1610,7 +1623,7 @@ METAL_CreateRenderer(SDL_Window * window, Uint32 flags)
         [mtldevice release];
 #endif
         SDL_free(renderer);
-        return NULL;
+        goto error;
     }
 
     // !!! FIXME: error checking on all of this.
@@ -1622,7 +1635,7 @@ METAL_CreateRenderer(SDL_Window * window, Uint32 flags)
 #endif
         SDL_Metal_DestroyView(view);
         SDL_free(renderer);
-        return NULL;
+        goto error;
     }
 
     renderer->driverdata = (void*)CFBridgingRetain(data);
@@ -1861,6 +1874,13 @@ METAL_CreateRenderer(SDL_Window * window, Uint32 flags)
 #endif
 
     return renderer;
+
+error:
+    if (changed_window) {
+        /* Uh oh, better try to put it back... */
+        SDL_RecreateWindow(window, window_flags);
+    }
+    return NULL;
 }}
 
 SDL_RenderDriver METAL_RenderDriver = {

src/video/SDL_video.c

@@ -1706,10 +1706,12 @@ SDL_RecreateWindow(SDL_Window * window, Uint32 flags)
         return -1;
     }
 
+    /*
     if ((window->flags & SDL_WINDOW_METAL) != (flags & SDL_WINDOW_METAL)) {
         SDL_SetError("Can't change SDL_WINDOW_METAL window flag");
         return -1;
     }
+    */
 
     if ((window->flags & SDL_WINDOW_VULKAN) && (flags & SDL_WINDOW_OPENGL)) {
         SDL_SetError("Vulkan and OpenGL not supported on same window");