Vulkan: support vktrace and renderdoc

src/backend/vulkan/VulkanBackend.cpp

@@ -347,6 +347,21 @@ namespace backend { namespace vulkan {
         std::vector<const char*> layersToRequest;
         std::vector<const char*> extensionsToRequest;
 
+        // vktrace works by instering a layer, so we need to explicitly enable it if it is present.
+        // Also it is good to put it in first position so that it doesn't see Vulkan calls inserted
+        // by other layers.
+        if (mGlobalInfo.vktrace) {
+            layersToRequest.push_back(kLayerNameLunargVKTrace);
+            usedKnobs->vktrace = true;
+        }
+        // RenderDoc installs a layer at the system level for its capture but we don't want to use
+        // it unless we are debugging in RenderDoc so we hide it behind a macro.
+#if defined(NXT_USE_RENDERDOC)
+        if (mGlobalInfo.renderDocCapture) {
+            layersToRequest.push_back(kLayerNameRenderDocCapture);
+            usedKnobs->renderDocCapture = true;
+        }
+#endif
 #if defined(NXT_ENABLE_ASSERTS)
         if (mGlobalInfo.standardValidation) {
             layersToRequest.push_back(kLayerNameLunargStandardValidation);

src/backend/vulkan/VulkanInfo.cpp

@@ -31,6 +31,8 @@ namespace {
 namespace backend { namespace vulkan {
 
     const char kLayerNameLunargStandardValidation[] = "VK_LAYER_LUNARG_standard_validation";
+    const char kLayerNameLunargVKTrace[] = "VK_LAYER_LUNARG_vktrace";
+    const char kLayerNameRenderDocCapture[] = "VK_LAYER_RENDERDOC_Capture";
 
     const char kExtensionNameExtDebugReport[] = "VK_EXT_debug_report";
     const char kExtensionNameKhrSurface[] = "VK_KHR_surface";
@@ -58,6 +60,12 @@ namespace backend { namespace vulkan {
                 if (IsLayerName(layer, kLayerNameLunargStandardValidation)) {
                     info->standardValidation = true;
                 }
+                if (IsLayerName(layer, kLayerNameLunargVKTrace)) {
+                    info->vktrace = true;
+                }
+                if (IsLayerName(layer, kLayerNameRenderDocCapture)) {
+                    info->renderDocCapture = true;
+                }
             }
         }
 

src/backend/vulkan/VulkanInfo.h

@@ -24,6 +24,8 @@ namespace backend { namespace vulkan {
     class Device;
 
     extern const char kLayerNameLunargStandardValidation[];
+    extern const char kLayerNameLunargVKTrace[];
+    extern const char kLayerNameRenderDocCapture[];
 
     extern const char kExtensionNameExtDebugReport[];
     extern const char kExtensionNameKhrSurface[];
@@ -33,6 +35,8 @@ namespace backend { namespace vulkan {
     struct VulkanGlobalKnobs {
         // Layers
         bool standardValidation = false;
+        bool vktrace = false;
+        bool renderDocCapture = false;
 
         // Extensions
         bool debugReport = false;