macOS build fixes

hecl/lib/Compilers.cpp

@@ -1,16 +1,18 @@
 #include "hecl/Compilers.hpp"
 #include "boo/graphicsdev/GLSLMacros.hpp"
 #include "logvisor/logvisor.hpp"
-#if BOO_HAS_VULKAN
 #include <glslang/Public/ShaderLang.h>
 #include <StandAlone/ResourceLimits.h>
 #include <SPIRV/GlslangToSpv.h>
 #include <SPIRV/disassemble.h>
-#endif
 #if _WIN32
 #include <d3dcompiler.h>
 extern pD3DCompile D3DCompilePROC;
 #endif
+#if __APPLE__
+#include <unistd.h>
+#include <memory>
+#endif
 
 namespace hecl
 {
@@ -40,13 +42,13 @@ template<typename P> struct ShaderCompiler {};
 template<> struct ShaderCompiler<PlatformType::OpenGL>
 {
     template<typename S>
-    static std::pair<std::shared_ptr<uint8_t[]>, size_t> Compile(std::string_view text)
+    static std::pair<StageBinaryData, size_t> Compile(std::string_view text)
     {
         std::string str = "#version 330\n";
         str += BOO_GLSL_BINDING_HEAD;
         str += text;
-        std::pair<std::shared_ptr<uint8_t[]>, size_t> ret(new uint8_t[str.size() + 1], str.size() + 1);
-        memcpy(ret.first.get(), str.data(), str.size() + 1);
+        std::pair<StageBinaryData, size_t> ret(MakeStageBinaryData(str.size() + 1), str.size() + 1);
+        memcpy(ret.first.get(), str.data(), ret.second);
         return ret;
     }
 };
@@ -64,7 +66,7 @@ template<> struct ShaderCompiler<PlatformType::Vulkan>
     };
 
     template<typename S>
-    static std::pair<std::shared_ptr<uint8_t[]>, size_t> Compile(std::string_view text)
+    static std::pair<StageBinaryData, size_t> Compile(std::string_view text)
     {
         EShLanguage lang = ShaderTypes[int(S::Enum)];
         const EShMessages messages = EShMessages(EShMsgSpvRules | EShMsgVulkanRules);
@@ -88,7 +90,7 @@ template<> struct ShaderCompiler<PlatformType::Vulkan>
 
         std::vector<unsigned int> out;
         glslang::GlslangToSpv(*prog.getIntermediate(lang), out);
-        std::pair<std::shared_ptr<uint8_t[]>, size_t> ret(new uint8_t[out.size() * 4], out.size() * 4);
+        std::pair<StageBinaryData, size_t> ret(MakeStageBinaryData(out.size() * 4), out.size() * 4);
         memcpy(ret.first.get(), out.data(), ret.second);
         return ret;
     }
@@ -112,7 +114,7 @@ template<> struct ShaderCompiler<PlatformType::D3D11>
 #define BOO_D3DCOMPILE_FLAG D3DCOMPILE_OPTIMIZATION_LEVEL3
 #endif
     template<typename S>
-    static std::pair<std::shared_ptr<uint8_t[]>, size_t> Compile(std::string_view text)
+    static std::pair<StageBinaryData, size_t> Compile(std::string_view text)
     {
         ComPtr<ID3DBlob> errBlob;
         ComPtr<ID3DBlob> blobOut;
@@ -123,13 +125,176 @@ template<> struct ShaderCompiler<PlatformType::D3D11>
             Log.report(logvisor::Fatal, "error compiling shader: %s", errBlob->GetBufferPointer());
             return {};
         }
-        std::pair<std::shared_ptr<uint8_t[]>, size_t> ret(new uint8_t[blobOut->GetBufferSize()], blobOut->GetBufferSize());
+        std::pair<StageBinaryData, size_t> ret(MakeStageBinaryData(blobOut->GetBufferSize()),
+                                               blobOut->GetBufferSize());
         memcpy(ret.first.get(), blobOut->GetBufferPointer(), blobOut->GetBufferSize());
         return ret;
     }
 };
 #endif
 
+#if __APPLE__
+template<> struct ShaderCompiler<PlatformType::Metal>
+{
+    static bool m_didCompilerSearch;
+    static bool m_hasCompiler;
+
+    static bool SearchForCompiler()
+    {
+        m_didCompilerSearch = true;
+        const char* no_metal_compiler = getenv("HECL_NO_METAL_COMPILER");
+        if (no_metal_compiler && atoi(no_metal_compiler))
+            return false;
+
+        pid_t pid = fork();
+        if (!pid)
+        {
+            execlp("xcrun", "xcrun", "-sdk", "macosx", "metal", NULL);
+            /* xcrun returns 72 if metal command not found;
+             * emulate that if xcrun not found */
+            exit(72);
+        }
+
+        int status, ret;
+        while ((ret = waitpid(pid, &status, 0)) < 0 && errno == EINTR) {}
+        if (ret < 0)
+            return false;
+        return WEXITSTATUS(status) == 1;
+    }
+
+    template<typename S>
+    static std::pair<StageBinaryData, size_t> Compile(std::string_view text)
+    {
+        if (!m_didCompilerSearch)
+            m_hasCompiler = SearchForCompiler();
+
+        std::string str = "#include <metal_stdlib>\n"
+                          "using namespace metal;\n";
+        str += text;
+        std::pair<StageBinaryData, size_t> ret;
+
+        if (!m_hasCompiler)
+        {
+            /* First byte unset to indicate source data */
+            ret.first = MakeStageBinaryData(str.size() + 2);
+            ret.first.get()[0] = 0;
+            ret.second = str.size() + 2;
+            memcpy(&ret.first.get()[1], str.data(), str.size() + 1);
+        }
+        else
+        {
+            int compilerOut[2];
+            int compilerIn[2];
+            pipe(compilerOut);
+            pipe(compilerIn);
+
+            pid_t pid = getpid();
+            const char* tmpdir = getenv("TMPDIR");
+            char libFile[1024];
+            snprintf(libFile, 1024, "%sboo_metal_shader%d.metallib", tmpdir, pid);
+
+            /* Pipe source write to compiler */
+            pid_t compilerPid = fork();
+            if (!compilerPid)
+            {
+                dup2(compilerIn[0], STDIN_FILENO);
+                dup2(compilerOut[1], STDOUT_FILENO);
+
+                close(compilerOut[0]);
+                close(compilerOut[1]);
+                close(compilerIn[0]);
+                close(compilerIn[1]);
+
+                execlp("xcrun", "xcrun", "-sdk", "macosx", "metal", "-o", "/dev/stdout", "-Wno-unused-variable",
+                       "-Wno-unused-const-variable", "-Wno-unused-function", "-c", "-x", "metal", "-", NULL);
+                fprintf(stderr, "execlp fail %s\n", strerror(errno));
+                exit(1);
+            }
+            close(compilerIn[0]);
+            close(compilerOut[1]);
+
+            /* Pipe compiler to linker */
+            pid_t linkerPid = fork();
+            if (!linkerPid)
+            {
+                dup2(compilerOut[0], STDIN_FILENO);
+
+                close(compilerOut[0]);
+                close(compilerIn[1]);
+
+                /* metallib doesn't like outputting to a pipe, so temp file will have to do */
+                execlp("xcrun", "xcrun", "-sdk", "macosx", "metallib", "-", "-o", libFile, NULL);
+                fprintf(stderr, "execlp fail %s\n", strerror(errno));
+                exit(1);
+            }
+            close(compilerOut[0]);
+
+            /* Stream in source */
+            const char* inPtr = str.data();
+            size_t inRem = str.size();
+            while (inRem)
+            {
+                ssize_t writeRes = write(compilerIn[1], inPtr, inRem);
+                if (writeRes < 0)
+                {
+                    fprintf(stderr, "write fail %s\n", strerror(errno));
+                    break;
+                }
+                inPtr += writeRes;
+                inRem -= writeRes;
+            }
+            close(compilerIn[1]);
+
+            /* Wait for completion */
+            int compilerStat, linkerStat;
+            while (waitpid(compilerPid, &compilerStat, 0) < 0)
+            {
+                if (errno == EINTR)
+                    continue;
+                Log.report(logvisor::Fatal, "waitpid fail %s", strerror(errno));
+                return {};
+            }
+
+            if (WEXITSTATUS(compilerStat))
+            {
+                Log.report(logvisor::Fatal, "compile fail");
+                return {};
+            }
+
+            while (waitpid(linkerPid, &linkerStat, 0) < 0)
+            {
+                if (errno == EINTR)
+                    continue;
+                Log.report(logvisor::Fatal, "waitpid fail %s", strerror(errno));
+                return {};
+            }
+
+            if (WEXITSTATUS(linkerStat))
+            {
+                Log.report(logvisor::Fatal, "link fail");
+                return {};
+            }
+
+            /* Copy temp file into buffer with first byte set to indicate binary data */
+            FILE* fin = fopen(libFile, "rb");
+            fseek(fin, 0, SEEK_END);
+            long libLen = ftell(fin);
+            fseek(fin, 0, SEEK_SET);
+            ret.first = MakeStageBinaryData(libLen + 1);
+            ret.first.get()[0] = 1;
+            ret.second = libLen + 1;
+            fread(&ret.first.get()[1], 1, libLen, fin);
+            fclose(fin);
+            unlink(libFile);
+        }
+
+        return ret;
+    }
+};
+bool ShaderCompiler<PlatformType::Metal>::m_didCompilerSearch = false;
+bool ShaderCompiler<PlatformType::Metal>::m_hasCompiler = false;
+#endif
+
 #if HECL_NOUVEAU_NX
 template<> struct ShaderCompiler<PlatformType::NX>
 {
@@ -147,16 +312,16 @@ template<> struct ShaderCompiler<PlatformType::NX>
 #endif
 
 template<typename P, typename S>
-std::pair<std::shared_ptr<uint8_t[]>, size_t> CompileShader(std::string_view text)
+std::pair<StageBinaryData, size_t> CompileShader(std::string_view text)
 {
     return ShaderCompiler<P>::template Compile<S>(text);
 }
 #define SPECIALIZE_COMPILE_SHADER(P) \
-template std::pair<std::shared_ptr<uint8_t[]>, size_t> CompileShader<P, PipelineStage::Vertex>(std::string_view text); \
-template std::pair<std::shared_ptr<uint8_t[]>, size_t> CompileShader<P, PipelineStage::Fragment>(std::string_view text); \
-template std::pair<std::shared_ptr<uint8_t[]>, size_t> CompileShader<P, PipelineStage::Geometry>(std::string_view text); \
-template std::pair<std::shared_ptr<uint8_t[]>, size_t> CompileShader<P, PipelineStage::Control>(std::string_view text); \
-template std::pair<std::shared_ptr<uint8_t[]>, size_t> CompileShader<P, PipelineStage::Evaluation>(std::string_view text);
+template std::pair<StageBinaryData, size_t> CompileShader<P, PipelineStage::Vertex>(std::string_view text); \
+template std::pair<StageBinaryData, size_t> CompileShader<P, PipelineStage::Fragment>(std::string_view text); \
+template std::pair<StageBinaryData, size_t> CompileShader<P, PipelineStage::Geometry>(std::string_view text); \
+template std::pair<StageBinaryData, size_t> CompileShader<P, PipelineStage::Control>(std::string_view text); \
+template std::pair<StageBinaryData, size_t> CompileShader<P, PipelineStage::Evaluation>(std::string_view text);
 SPECIALIZE_COMPILE_SHADER(PlatformType::OpenGL)
 SPECIALIZE_COMPILE_SHADER(PlatformType::Vulkan)
 #if _WIN32
@@ -169,127 +334,4 @@ SPECIALIZE_COMPILE_SHADER(PlatformType::Metal)
 SPECIALIZE_COMPILE_SHADER(PlatformType::NX)
 #endif
 
-#if BOO_HAS_METAL
-static int HasMetalCompiler = -1;
-
-static void CheckForMetalCompiler()
-{
-    pid_t pid = fork();
-    if (!pid)
-    {
-        execlp("xcrun", "xcrun", "-sdk", "macosx", "metal", NULL);
-        /* xcrun returns 72 if metal command not found;
-         * emulate that if xcrun not found */
-        exit(72);
-    }
-
-    int status, ret;
-    while ((ret = waitpid(pid, &status, 0)) < 0 && errno == EINTR) {}
-    if (ret < 0)
-        HasMetalCompiler = 0;
-    else
-        HasMetalCompiler = WEXITSTATUS(status) == 1;
-}
-
-template<>
-std::vector<uint8_t> CompileShader<PlatformType::Metal>(std::string_view text, PipelineStage stage)
-{
-    if (HasMetalCompiler == -1)
-        CheckForMetalCompiler();
-
-    std::vector<uint8_t> blobOut;
-    if (!HasMetalCompiler)
-    {
-        /* Cache the source if there's no compiler */
-        size_t sourceLen = strlen(source);
-
-        /* First byte unset to indicate source data */
-        blobOut.resize(sourceLen + 2);
-        memcpy(&blobOut[1], source, sourceLen);
-    }
-    else
-    {
-        /* Cache the binary otherwise */
-        int compilerOut[2];
-        int compilerIn[2];
-        pipe(compilerOut);
-        pipe(compilerIn);
-
-        /* Pipe source write to compiler */
-        pid_t compilerPid = fork();
-        if (!compilerPid)
-        {
-            dup2(compilerIn[0], STDIN_FILENO);
-            dup2(compilerOut[1], STDOUT_FILENO);
-
-            close(compilerOut[0]);
-            close(compilerOut[1]);
-            close(compilerIn[0]);
-            close(compilerIn[1]);
-
-            execlp("xcrun", "xcrun", "-sdk", "macosx", "metal", "-o", "/dev/stdout", "-Wno-unused-variable",
-                   "-Wno-unused-const-variable", "-Wno-unused-function", "-x", "metal", "-", NULL);
-            fprintf(stderr, "execlp fail %s\n", strerror(errno));
-            exit(1);
-        }
-        close(compilerIn[0]);
-        close(compilerOut[1]);
-
-        /* Pipe compiler to linker */
-        pid_t linkerPid = fork();
-        if (!linkerPid)
-        {
-            dup2(compilerOut[0], STDIN_FILENO);
-
-            close(compilerOut[0]);
-            close(compilerIn[1]);
-
-            /* metallib doesn't like outputting to a pipe, so temp file will have to do */
-            execlp("xcrun", "xcrun", "-sdk", "macosx", "metallib", "-", "-o", m_libfile, NULL);
-            fprintf(stderr, "execlp fail %s\n", strerror(errno));
-            exit(1);
-        }
-        close(compilerOut[0]);
-
-        /* Stream in source */
-        const char* inPtr = source;
-        size_t inRem = strlen(source);
-        while (inRem)
-        {
-            ssize_t writeRes = write(compilerIn[1], inPtr, inRem);
-            if (writeRes < 0)
-            {
-                fprintf(stderr, "write fail %s\n", strerror(errno));
-                break;
-            }
-            inPtr += writeRes;
-            inRem -= writeRes;
-        }
-        close(compilerIn[1]);
-
-        /* Wait for completion */
-        int compilerStat, linkerStat;
-        if (waitpid(compilerPid, &compilerStat, 0) < 0 || waitpid(linkerPid, &linkerStat, 0) < 0)
-        {
-            fprintf(stderr, "waitpid fail %s\n", strerror(errno));
-            return {};
-        }
-
-        if (WEXITSTATUS(compilerStat) || WEXITSTATUS(linkerStat))
-            return {};
-
-        /* Copy temp file into buffer with first byte set to indicate binary data */
-        FILE* fin = fopen(m_libfile, "rb");
-        fseek(fin, 0, SEEK_END);
-        long libLen = ftell(fin);
-        fseek(fin, 0, SEEK_SET);
-        blobOut.resize(libLen + 1);
-        blobOut[0] = 1;
-        fread(&blobOut[1], 1, libLen, fin);
-        fclose(fin);
-    }
-    return blobOut;
-}
-#endif
-
 }