More reliable shader cache; update submodules

hecl/extern/athena

@@ -1 +1 @@
-Subproject commit 750d23c686e1f571643c019ae0ed6734c8057194
+Subproject commit b43c4af3a777d6baf43a6108764648e52f3b7ea6

hecl/extern/boo

@@ -1 +1 @@
-Subproject commit 8b923214fd178cf96d61159a4cf099348a230e92
+Subproject commit c77fe943133c4d797835490920c3444d8edbf795

hecl/include/hecl/Runtime.hpp

@@ -187,7 +187,7 @@ protected:
                                                        const std::vector<ShaderCacheExtensions::ExtensionSlot>& extensionSlots,
                                                        boo::IGraphicsDataFactory::Context& ctx,
                                                        FReturnExtensionShader returnFunc)=0;
-    virtual void buildExtendedShaderFromCache(const ShaderCachedData& data,
+    virtual bool buildExtendedShaderFromCache(const ShaderCachedData& data,
                                               const std::vector<ShaderCacheExtensions::ExtensionSlot>& extensionSlots,
                                               boo::IGraphicsDataFactory::Context& ctx,
                                               FReturnExtensionShader returnFunc)=0;

hecl/lib/Backend/GLSL.cpp

@@ -379,13 +379,16 @@ struct GLSLBackendFactory : IShaderBackendFactory
                                                boo::IGraphicsDataFactory::Context& ctx)
     {
         const ShaderTag& tag = data.m_tag;
-        athena::io::MemoryReader r(data.m_data.get(), data.m_sz);
+        athena::io::MemoryReader r(data.m_data.get(), data.m_sz, false, false);
         atUint8 texMapEnd = r.readUByte();
         boo::BlendFactor blendSrc = boo::BlendFactor(r.readUByte());
         boo::BlendFactor blendDst = boo::BlendFactor(r.readUByte());
         std::string vertSource = r.readString();
         std::string fragSource = r.readString();
 
+        if (r.hasError())
+            return nullptr;
+
         if (texMapEnd > 8)
             Log.report(logvisor::Fatal, "maximum of 8 texture maps supported");
 
@@ -463,17 +466,20 @@ struct GLSLBackendFactory : IShaderBackendFactory
         return dataOut;
     }
 
-    void buildExtendedShaderFromCache(const ShaderCachedData& data,
+    bool buildExtendedShaderFromCache(const ShaderCachedData& data,
                                       const std::vector<ShaderCacheExtensions::ExtensionSlot>& extensionSlots,
                                       boo::IGraphicsDataFactory::Context& ctx,
                                       FReturnExtensionShader returnFunc)
     {
         const ShaderTag& tag = data.m_tag;
-        athena::io::MemoryReader r(data.m_data.get(), data.m_sz);
+        athena::io::MemoryReader r(data.m_data.get(), data.m_sz, false, false);
         atUint8 texMapEnd = r.readUByte();
         hecl::Backend::BlendFactor blendSrc = hecl::Backend::BlendFactor(r.readUByte());
         hecl::Backend::BlendFactor blendDst = hecl::Backend::BlendFactor(r.readUByte());
 
+        if (r.hasError())
+            return false;
+
         if (texMapEnd > 8)
             Log.report(logvisor::Fatal, "maximum of 8 texture maps supported");
 
@@ -489,6 +495,10 @@ struct GLSLBackendFactory : IShaderBackendFactory
 
             std::string vertSource = r.readString();
             std::string fragSource = r.readString();
+
+            if (r.hasError())
+                return false;
+
             boo::IShaderPipeline* ret =
             static_cast<boo::GLDataFactory::Context&>(ctx).
                     newShaderPipeline(vertSource.c_str(), fragSource.c_str(),
@@ -501,6 +511,8 @@ struct GLSLBackendFactory : IShaderBackendFactory
                 Log.report(logvisor::Fatal, "unable to build shader");
             returnFunc(ret);
         }
+
+        return true;
     }
 };
 
@@ -589,7 +601,7 @@ struct SPIRVBackendFactory : IShaderBackendFactory
                                                boo::IGraphicsDataFactory::Context& ctx)
     {
         const ShaderTag& tag = data.m_tag;
-        athena::io::MemoryReader r(data.m_data.get(), data.m_sz);
+        athena::io::MemoryReader r(data.m_data.get(), data.m_sz, false, false);
         size_t texCount = size_t(r.readByte());
         boo::BlendFactor blendSrc = boo::BlendFactor(r.readUByte());
         boo::BlendFactor blendDst = boo::BlendFactor(r.readUByte());
@@ -609,6 +621,9 @@ struct SPIRVBackendFactory : IShaderBackendFactory
         if (pipelineSz)
             r.readUBytesToBuf(pipelineBlob.data(), pipelineSz);
 
+        if (r.hasError())
+            return nullptr;
+
         boo::IShaderPipeline* ret =
         static_cast<boo::VulkanDataFactory::Context&>(ctx).
                 newShaderPipeline(nullptr, nullptr,
@@ -712,7 +727,7 @@ struct SPIRVBackendFactory : IShaderBackendFactory
         return dataOut;
     }
 
-    void buildExtendedShaderFromCache(const ShaderCachedData& data,
+    bool buildExtendedShaderFromCache(const ShaderCachedData& data,
                                       const std::vector<ShaderCacheExtensions::ExtensionSlot>& extensionSlots,
                                       boo::IGraphicsDataFactory::Context& ctx,
                                       FReturnExtensionShader returnFunc)
@@ -723,6 +738,9 @@ struct SPIRVBackendFactory : IShaderBackendFactory
         hecl::Backend::BlendFactor blendSrc = hecl::Backend::BlendFactor(r.readUByte());
         hecl::Backend::BlendFactor blendDst = hecl::Backend::BlendFactor(r.readUByte());
 
+        if (r.hasError())
+            return false;
+
         for (const ShaderCacheExtensions::ExtensionSlot& slot : extensionSlots)
         {
             atUint32 vertSz = r.readUint32Big();
@@ -740,6 +758,9 @@ struct SPIRVBackendFactory : IShaderBackendFactory
             if (pipelineSz)
                 r.readUBytesToBuf(pipelineBlob.data(), pipelineSz);
 
+            if (r.hasError())
+                return false;
+
             boo::IShaderPipeline* ret =
             static_cast<boo::VulkanDataFactory::Context&>(ctx).
                     newShaderPipeline(nullptr, nullptr,
@@ -753,6 +774,8 @@ struct SPIRVBackendFactory : IShaderBackendFactory
                 Log.report(logvisor::Fatal, "unable to build shader");
             returnFunc(ret);
         }
+
+        return true;
     }
 };
 

hecl/lib/Runtime/ShaderCacheManager.cpp

@@ -368,9 +368,15 @@ ShaderCacheManager::buildShader(const ShaderTag& tag, const std::string& source,
                                 const std::string& diagName,
                                 boo::IGraphicsDataFactory::Context& ctx)
 {
+    boo::IShaderPipeline* ret;
     ShaderCachedData foundData = lookupData(tag);
     if (foundData)
-        return buildFromCache(foundData, ctx);
+    {
+        ret = buildFromCache(foundData, ctx);
+        if (ret)
+            return ret;
+        Log.report(logvisor::Warning, "invalid cache read, rebuilding shader '%s'", diagName.c_str());
+    }
     hecl::Frontend::IR ir = FE.compileSource(source, diagName);
     return buildShader(tag, ir, diagName, ctx);
 }
@@ -380,11 +386,16 @@ ShaderCacheManager::buildShader(const ShaderTag& tag, const hecl::Frontend::IR&
                                 const std::string& diagName,
                                 boo::IGraphicsDataFactory::Context& ctx)
 {
+    boo::IShaderPipeline* ret;
     ShaderCachedData foundData = lookupData(tag);
     if (foundData)
-        return buildFromCache(foundData, ctx);
+    {
+        ret = buildFromCache(foundData, ctx);
+        if (ret)
+            return ret;
+        Log.report(logvisor::Warning, "invalid cache read, rebuilding shader '%s'", diagName.c_str());
+    }
     FE.getDiagnostics().reset(diagName);
-    boo::IShaderPipeline* ret;
     addData(m_factory->buildShaderFromIR(tag, ir, FE.getDiagnostics(), ctx, ret));
     return ret;
 }
@@ -395,8 +406,9 @@ ShaderCacheManager::buildExtendedFromCache(const ShaderCachedData& foundData,
 {
     std::vector<boo::IShaderPipeline*> shaders;
     shaders.reserve(m_extensions.m_extensionSlots.size());
-    m_factory->buildExtendedShaderFromCache(foundData, m_extensions.m_extensionSlots, ctx,
+    if (!m_factory->buildExtendedShaderFromCache(foundData, m_extensions.m_extensionSlots, ctx,
-    [&](boo::IShaderPipeline* shader){shaders.push_back(shader);});
+    [&](boo::IShaderPipeline* shader){shaders.push_back(shader);}))
+        return {};
     if (shaders.size() != m_extensions.m_extensionSlots.size())
         Log.report(logvisor::Fatal, "buildShaderFromCache returned %" PRISize " times, expected %" PRISize,
                    shaders.size(), m_extensions.m_extensionSlots.size());
@@ -408,9 +420,15 @@ ShaderCacheManager::buildExtendedShader(const ShaderTag& tag, const std::string&
                                         const std::string& diagName,
                                         boo::IGraphicsDataFactory::Context& ctx)
 {
+    std::vector<boo::IShaderPipeline*> shaders;
     ShaderCachedData foundData = lookupData(tag);
     if (foundData)
-        return buildExtendedFromCache(foundData, ctx);
+    {
+        shaders = buildExtendedFromCache(foundData, ctx);
+        if (shaders.size())
+            return shaders;
+        Log.report(logvisor::Warning, "invalid cache read, rebuilding shader '%s'", diagName.c_str());
+    }
     hecl::Frontend::IR ir = FE.compileSource(source, diagName);
     return buildExtendedShader(tag, ir, diagName, ctx);
 }
@@ -420,10 +438,15 @@ ShaderCacheManager::buildExtendedShader(const ShaderTag& tag, const hecl::Fronte
                                         const std::string& diagName,
                                         boo::IGraphicsDataFactory::Context& ctx)
 {
+    std::vector<boo::IShaderPipeline*> shaders;
     ShaderCachedData foundData = lookupData(tag);
     if (foundData)
-        return buildExtendedFromCache(foundData, ctx);
+    {
-    std::vector<boo::IShaderPipeline*> shaders;
+        shaders = buildExtendedFromCache(foundData, ctx);
+        if (shaders.size())
+            return shaders;
+        Log.report(logvisor::Warning, "invalid cache read, rebuilding shader '%s'", diagName.c_str());
+    }
     shaders.reserve(m_extensions.m_extensionSlots.size());
     FE.getDiagnostics().reset(diagName);
     ShaderCachedData data =