Integrate hecl ClientProcess into resource factory

Editor/ProjectManager.hpp

@@ -31,9 +31,14 @@ public:
     bool newProject(const hecl::SystemString& path);
     bool openProject(const hecl::SystemString& path);
     bool extractGame(const hecl::SystemString& path);
-
     bool saveProject();
 
+    void asyncIdle() {m_factoryMP1.AsyncIdle();}
+    void shutdown()
+    {
+        m_clientProc.shutdown();
+        m_factoryMP1.Shutdown();
+    }
 };
 
 }

Editor/ProjectResourceFactoryBase.cpp

@@ -44,24 +44,29 @@ void ProjectResourceFactoryBase::ReadCatalog(const hecl::ProjectPath& catalogPat
         {
             std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
             m_catalogNameToTag[p.first] = pathTag;
+#if 0
+            fprintf(stderr, "%s %s %08X\n",
+                    p.first.c_str(),
+                    pathTag.type.toString().c_str(), uint32_t(pathTag.id));
+#endif
         }
     }
 }
 
-void ProjectResourceFactoryBase::BackgroundIndexRecursiveProc(const hecl::SystemString& path, int level)
+void ProjectResourceFactoryBase::BackgroundIndexRecursiveProc(const hecl::ProjectPath& dir, int level)
 {
-    hecl::DirectoryEnumerator dEnum(path,
+    hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(),
                                     hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted,
                                     false, false, true);
 
     /* Enumerate all items */
     for (const hecl::DirectoryEnumerator::Entry& ent : dEnum)
     {
+        hecl::ProjectPath path(dir, ent.m_name);
         if (ent.m_isDir)
-            BackgroundIndexRecursiveProc(ent.m_path, level+1);
+            BackgroundIndexRecursiveProc(path, level+1);
         else
         {
-            hecl::ProjectPath path(path, ent.m_name);
             if (path.getPathType() != hecl::ProjectPath::Type::File)
                 continue;
 
@@ -78,6 +83,11 @@ void ProjectResourceFactoryBase::BackgroundIndexRecursiveProc(const hecl::System
             {
                 std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
                 m_tagToPath[pathTag] = path;
+#if 0
+                fprintf(stderr, "%s %08X %s\n",
+                        pathTag.type.toString().c_str(), uint32_t(pathTag.id),
+                        path.getRelativePathUTF8().c_str());
+#endif
             }
         }
 
@@ -90,7 +100,7 @@ void ProjectResourceFactoryBase::BackgroundIndexRecursiveProc(const hecl::System
 void ProjectResourceFactoryBase::BackgroundIndexProc()
 {
     hecl::ProjectPath specRoot(m_proj->getProjectWorkingPath(), m_origSpec->m_name);
-    BackgroundIndexRecursiveProc(specRoot.getAbsolutePath(), 0);
+    BackgroundIndexRecursiveProc(specRoot, 0);
     m_backgroundRunning = false;
     m_backgroundBlender = std::experimental::nullopt;
 }
@@ -105,7 +115,7 @@ void ProjectResourceFactoryBase::CancelBackgroundIndex()
 }
 
 void ProjectResourceFactoryBase::BeginBackgroundIndex
-    (const hecl::Database::Project& proj,
+    (hecl::Database::Project& proj,
      const hecl::Database::DataSpecEntry& origSpec,
      const hecl::Database::DataSpecEntry& pcSpec)
 {
@@ -114,14 +124,31 @@ void ProjectResourceFactoryBase::BeginBackgroundIndex
     m_proj = &proj;
     m_origSpec = &origSpec;
     m_pcSpec = &pcSpec;
+    m_cookSpec.reset(pcSpec.m_factory(proj, hecl::Database::DataSpecTool::Cook));
     m_backgroundRunning = true;
     m_backgroundIndexTh =
         std::thread(std::bind(&ProjectResourceFactoryBase::BackgroundIndexProc, this));
 }
 
-CFactoryFnReturn ProjectResourceFactoryBase::MakeObject(const SObjectTag& tag,
+hecl::ProjectPath ProjectResourceFactoryBase::GetCookedPath(const hecl::ProjectPath& working,
-                                                        const hecl::ProjectPath& path,
+                                                            bool pcTarget) const
-                                                        const CVParamTransfer& paramXfer)
+{
+    const hecl::Database::DataSpecEntry* spec = m_origSpec;
+    if (pcTarget)
+        spec = m_cookSpec->overrideDataSpec(working, m_pcSpec);
+    if (!spec)
+        return {};
+    return working.getCookedPath(*spec);
+}
+
+bool ProjectResourceFactoryBase::SyncCook(const hecl::ProjectPath& working)
+{
+    return m_clientProc.syncCook(working) == 0;
+}
+
+CFactoryFnReturn ProjectResourceFactoryBase::SyncMakeObject(const SObjectTag& tag,
+                                                            const hecl::ProjectPath& path,
+                                                            const CVParamTransfer& paramXfer)
 {
     /* Ensure requested resource is on the filesystem */
     if (path.getPathType() != hecl::ProjectPath::Type::File)
@@ -132,13 +159,14 @@ CFactoryFnReturn ProjectResourceFactoryBase::MakeObject(const SObjectTag& tag,
     }
 
     /* Get cooked representation path */
-    hecl::ProjectPath cooked = GetCookedPath(tag, path, true);
+    hecl::ProjectPath cooked = GetCookedPath(path, true);
 
     /* Perform mod-time comparison */
     if (cooked.getPathType() != hecl::ProjectPath::Type::File ||
         cooked.getModtime() < path.getModtime())
     {
-        if (!DoCook(tag, path, cooked, true))
+        /* Do a blocking cook here */
+        if (!SyncCook(path))
         {
             Log.report(logvisor::Error, _S("unable to cook resource path '%s'"),
                        path.getAbsolutePath().c_str());
@@ -159,6 +187,76 @@ CFactoryFnReturn ProjectResourceFactoryBase::MakeObject(const SObjectTag& tag,
     return m_factoryMgr.MakeObject(tag, fr, paramXfer);
 }
 
+void ProjectResourceFactoryBase::AsyncTask::EnsurePath(const hecl::ProjectPath& path)
+{
+    if (!m_workingPath)
+    {
+        m_workingPath = path;
+
+        /* Ensure requested resource is on the filesystem */
+        if (path.getPathType() != hecl::ProjectPath::Type::File)
+        {
+            Log.report(logvisor::Error, _S("unable to find resource path '%s'"),
+                       path.getAbsolutePath().c_str());
+            m_failed = true;
+            return;
+        }
+
+        /* Get cooked representation path */
+        m_cookedPath = m_parent.GetCookedPath(path, true);
+
+        /* Perform mod-time comparison */
+        if (m_cookedPath.getPathType() != hecl::ProjectPath::Type::File ||
+            m_cookedPath.getModtime() < path.getModtime())
+        {
+            /* Start a background cook here */
+            m_cookTransaction = m_parent.m_clientProc.addCookTransaction(path);
+            return;
+        }
+
+        CookComplete();
+    }
+}
+
+void ProjectResourceFactoryBase::AsyncTask::CookComplete()
+{
+    /* Ensure cooked rep is on the filesystem */
+    athena::io::FileReader fr(m_cookedPath.getAbsolutePath(), 32 * 1024, false);
+    if (fr.hasError())
+    {
+        Log.report(logvisor::Error, _S("unable to open cooked resource path '%s'"),
+                   m_cookedPath.getAbsolutePath().c_str());
+        m_failed = true;
+        return;
+    }
+
+    /* Ready for buffer transaction at this point */
+    x14_resSize = fr.length();
+    x10_loadBuffer.reset(new u8[x14_resSize]);
+    m_bufTransaction = m_parent.m_clientProc.addBufferTransaction(m_cookedPath,
+                                                                  x10_loadBuffer.get(),
+                                                                  x14_resSize, 0);
+}
+
+bool ProjectResourceFactoryBase::AsyncTask::AsyncPump()
+{
+    if (m_failed)
+        return true;
+
+    if (m_bufTransaction)
+    {
+        if (m_bufTransaction->m_complete)
+            return true;
+    }
+    else if (m_cookTransaction)
+    {
+        if (m_cookTransaction->m_complete)
+            CookComplete();
+    }
+
+    return m_failed;
+}
+
 std::unique_ptr<urde::IObj> ProjectResourceFactoryBase::Build(const urde::SObjectTag& tag,
                                                               const urde::CVParamTransfer& paramXfer)
 {
@@ -181,7 +279,7 @@ std::unique_ptr<urde::IObj> ProjectResourceFactoryBase::Build(const urde::SObjec
             return {};
     }
 
-    return MakeObject(tag, search->second, paramXfer);
+    return SyncMakeObject(tag, search->second, paramXfer);
 }
 
 void ProjectResourceFactoryBase::BuildAsync(const urde::SObjectTag& tag,
@@ -190,7 +288,7 @@ void ProjectResourceFactoryBase::BuildAsync(const urde::SObjectTag& tag,
 {
     if (m_asyncLoadList.find(tag) != m_asyncLoadList.end())
         return;
-    m_asyncLoadList[tag] = CResFactory::SLoadingData(tag, objOut, paramXfer);
+    m_asyncLoadList.emplace(std::make_pair(tag, AsyncTask(*this, tag, objOut, paramXfer)));
 }
 
 void ProjectResourceFactoryBase::CancelBuild(const urde::SObjectTag& tag)
@@ -250,8 +348,13 @@ const urde::SObjectTag* ProjectResourceFactoryBase::GetResourceIdByName(const ch
 
 void ProjectResourceFactoryBase::AsyncIdle()
 {
+    /* Consume completed transactions, they will be processed this cycle at the latest */
+    std::list<std::unique_ptr<hecl::ClientProcess::Transaction>> completed;
+    m_clientProc.swapCompletedQueue(completed);
+
+    /* Begin self-profiling loop */
     std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now();
-    for (auto it=m_asyncLoadList.begin() ; it != m_asyncLoadList.end() ; ++it)
+    for (auto it=m_asyncLoadList.begin() ; it != m_asyncLoadList.end() ;)
     {
         /* Allow 8 milliseconds (roughly 1/2 frame-time) for each async build cycle */
         if (std::chrono::duration_cast<std::chrono::milliseconds>(
@@ -260,7 +363,7 @@ void ProjectResourceFactoryBase::AsyncIdle()
 
         /* Ensure requested resource is in the index */
         std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
-        CResFactory::SLoadingData& data = it->second;
+        AsyncTask& data = it->second;
         auto search = m_tagToPath.find(data.x0_tag);
         if (search == m_tagToPath.end())
         {
@@ -272,11 +375,18 @@ void ProjectResourceFactoryBase::AsyncIdle()
             }
             continue;
         }
+        data.EnsurePath(search->second);
 
-        /* Perform build (data not actually loaded asynchronously for now) */
+        /* Pump load pipeline (cooking if needed) */
-        CFactoryFnReturn ret = MakeObject(data.x0_tag, search->second, data.x18_cvXfer);
+        if (data.AsyncPump())
-        *data.xc_targetPtr = ret.release();
+        {
-        it = m_asyncLoadList.erase(it);
+            /* Load complete, build resource */
+            athena::io::MemoryReader mr(data.x10_loadBuffer.get(), data.x14_resSize);
+            *data.xc_targetPtr = m_factoryMgr.MakeObject(data.x0_tag, mr, data.x18_cvXfer).release();
+            it = m_asyncLoadList.erase(it);
+            continue;
+        }
+        ++it;
     }
 }
 

Editor/ProjectResourceFactoryBase.hpp

@@ -2,6 +2,7 @@
 #define URDE_PROJECT_RESOURCE_FACTORY_BASE_HPP
 
 #include "hecl/ClientProcess.hpp"
+#include "hecl/Database.hpp"
 #include "Runtime/IFactory.hpp"
 #include "Runtime/CFactoryMgr.hpp"
 #include "Runtime/CResFactory.hpp"
@@ -24,6 +25,8 @@ protected:
     const hecl::Database::Project* m_proj = nullptr;
     const hecl::Database::DataSpecEntry* m_origSpec = nullptr;
     const hecl::Database::DataSpecEntry* m_pcSpec = nullptr;
+    /* Used to resolve cooked paths */
+    std::unique_ptr<hecl::Database::IDataSpec> m_cookSpec;
     urde::CFactoryMgr m_factoryMgr;
 
     std::experimental::optional<hecl::BlenderConnection> m_backgroundBlender;
@@ -31,27 +34,48 @@ protected:
     std::mutex m_backgroundIndexMutex;
     bool m_backgroundRunning = false;
 
-    std::unordered_map<SObjectTag, CResFactory::SLoadingData> m_asyncLoadList;
+    struct AsyncTask
+    {
+        ProjectResourceFactoryBase& m_parent;
+
+        SObjectTag x0_tag;
+        IDvdRequest* x8_dvdReq = nullptr;
+        IObj** xc_targetPtr = nullptr;
+        std::unique_ptr<u8[]> x10_loadBuffer;
+        u32 x14_resSize = 0;
+        CVParamTransfer x18_cvXfer;
+
+        hecl::ProjectPath m_workingPath;
+        hecl::ProjectPath m_cookedPath;
+        const hecl::ClientProcess::CookTransaction* m_cookTransaction = nullptr;
+        const hecl::ClientProcess::BufferTransaction* m_bufTransaction = nullptr;
+        bool m_failed = false;
+
+        AsyncTask(ProjectResourceFactoryBase& parent, const SObjectTag& tag,
+                  IObj** ptr, const CVParamTransfer& xfer)
+        : m_parent(parent), x0_tag(tag), xc_targetPtr(ptr), x18_cvXfer(xfer) {}
+
+        void EnsurePath(const hecl::ProjectPath& path);
+        void CookComplete();
+        bool AsyncPump();
+    };
+    std::unordered_map<SObjectTag, AsyncTask> m_asyncLoadList;
 
     virtual SObjectTag TagFromPath(const hecl::ProjectPath& path) const=0;
 
     hecl::BlenderConnection& GetBackgroundBlender() const;
     void ReadCatalog(const hecl::ProjectPath& catalogPath);
-    void BackgroundIndexRecursiveProc(const hecl::SystemString& path, int level);
+    void BackgroundIndexRecursiveProc(const hecl::ProjectPath& path, int level);
     void BackgroundIndexProc();
     void CancelBackgroundIndex();
-    void BeginBackgroundIndex(const hecl::Database::Project& proj,
+    void BeginBackgroundIndex(hecl::Database::Project& proj,
                               const hecl::Database::DataSpecEntry& origSpec,
                               const hecl::Database::DataSpecEntry& pcSpec);
 
-    virtual hecl::ProjectPath GetCookedPath(const SObjectTag& tag,
+    hecl::ProjectPath GetCookedPath(const hecl::ProjectPath& working, bool pcTarget) const;
-                                            const hecl::ProjectPath& working,
+    bool SyncCook(const hecl::ProjectPath& working);
-                                            bool pcTarget) const=0;
+    CFactoryFnReturn SyncMakeObject(const SObjectTag& tag, const hecl::ProjectPath& path,
-    virtual bool DoCook(const SObjectTag& tag, const hecl::ProjectPath& working,
+                                    const CVParamTransfer& paramXfer);
-                        const hecl::ProjectPath& cooked,
-                        bool pcTarget)=0;
-    CFactoryFnReturn MakeObject(const SObjectTag& tag, const hecl::ProjectPath& path,
-                                const CVParamTransfer& paramXfer);
 
 public:
     ProjectResourceFactoryBase(hecl::ClientProcess& clientProc) : m_clientProc(clientProc) {}
@@ -62,6 +86,7 @@ public:
     const urde::SObjectTag* GetResourceIdByName(const char*) const;
 
     void AsyncIdle();
+    void Shutdown() {CancelBackgroundIndex();}
 };
 
 }

Editor/ProjectResourceFactoryMP1.cpp

@@ -30,7 +30,7 @@ ProjectResourceFactoryMP1::ProjectResourceFactoryMP1(hecl::ClientProcess& client
     m_factoryMgr.AddFactory(FOURCC('FONT'), urde::FRasterFontFactory);
 }
 
-void ProjectResourceFactoryMP1::IndexMP1Resources(const hecl::Database::Project& proj)
+void ProjectResourceFactoryMP1::IndexMP1Resources(hecl::Database::Project& proj)
 {
     BeginBackgroundIndex(proj, DataSpec::SpecEntMP1, DataSpec::SpecEntMP1PC);
 }
@@ -46,26 +46,26 @@ SObjectTag ProjectResourceFactoryMP1::TagFromPath(const hecl::ProjectPath& path)
         switch (conn.getBlendType())
         {
         case hecl::BlenderConnection::BlendType::Mesh:
-            return {SBIG('CMDL'), path.hash()};
+            return {SBIG('CMDL'), path.hash().val32()};
         case hecl::BlenderConnection::BlendType::Actor:
-            return {SBIG('ANCS'), path.hash()};
+            return {SBIG('ANCS'), path.hash().val32()};
         case hecl::BlenderConnection::BlendType::Area:
-            return {SBIG('MREA'), path.hash()};
+            return {SBIG('MREA'), path.hash().val32()};
         case hecl::BlenderConnection::BlendType::World:
-            return {SBIG('MLVL'), path.hash()};
+            return {SBIG('MLVL'), path.hash().val32()};
         case hecl::BlenderConnection::BlendType::MapArea:
-            return {SBIG('MAPA'), path.hash()};
+            return {SBIG('MAPA'), path.hash().val32()};
         case hecl::BlenderConnection::BlendType::MapUniverse:
-            return {SBIG('MAPU'), path.hash()};
+            return {SBIG('MAPU'), path.hash().val32()};
         case hecl::BlenderConnection::BlendType::Frame:
-            return {SBIG('FRME'), path.hash()};
+            return {SBIG('FRME'), path.hash().val32()};
         default:
             return {};
         }
     }
     else if (hecl::IsPathPNG(path))
     {
-        return {SBIG('TXTR'), path.hash()};
+        return {SBIG('TXTR'), path.hash().val32()};
     }
     else if (hecl::IsPathYAML(path))
     {
@@ -75,10 +75,6 @@ SObjectTag ProjectResourceFactoryMP1::TagFromPath(const hecl::ProjectPath& path)
 
         athena::io::YAMLDocReader reader;
         yaml_parser_set_input_file(reader.getParser(), fp);
-        bool res = reader.parse();
-        fclose(fp);
-        if (!res)
-            return {};
 
         SObjectTag resTag;
         if (reader.ClassTypeOperation([&](const char* className) -> bool
@@ -96,47 +92,13 @@ SObjectTag ProjectResourceFactoryMP1::TagFromPath(const hecl::ProjectPath& path)
             return false;
         }))
         {
-            resTag.id = path.hash();
+            resTag.id = path.hash().val32();
+            fclose(fp);
             return resTag;
         }
+        fclose(fp);
     }
     return {};
 }
 
-hecl::ProjectPath ProjectResourceFactoryMP1::GetCookedPath(const SObjectTag& tag,
-                                                           const hecl::ProjectPath& working,
-                                                           bool pcTarget) const
-{
-    if (!pcTarget)
-        return working.getCookedPath(*m_origSpec);
-
-    switch (tag.type)
-    {
-    case SBIG('TXTR'):
-    case SBIG('CMDL'):
-    case SBIG('MREA'):
-        return working.getCookedPath(*m_pcSpec);
-    default: break;
-    }
-
-    return working.getCookedPath(*m_origSpec);
-}
-
-bool ProjectResourceFactoryMP1::DoCook(const SObjectTag& tag,
-                                       const hecl::ProjectPath& working,
-                                       const hecl::ProjectPath& cooked,
-                                       bool pcTarget)
-{
-    switch (tag.type)
-    {
-    case SBIG('TXTR'):
-        if (pcTarget)
-            return DataSpec::TXTR::CookPC(working, cooked);
-        else
-            return DataSpec::TXTR::Cook(working, cooked);
-    default: break;
-    }
-    return false;
-}
-
 }

Editor/ProjectResourceFactoryMP1.hpp

@@ -10,14 +10,8 @@ class ProjectResourceFactoryMP1 : public ProjectResourceFactoryBase
 {
 public:
     ProjectResourceFactoryMP1(hecl::ClientProcess& clientProc);
-    void IndexMP1Resources(const hecl::Database::Project& proj);
+    void IndexMP1Resources(hecl::Database::Project& proj);
     SObjectTag TagFromPath(const hecl::ProjectPath& path) const;
-    hecl::ProjectPath GetCookedPath(const SObjectTag& tag,
-                                    const hecl::ProjectPath& working,
-                                    bool pcTarget) const;
-    bool DoCook(const SObjectTag& tag, const hecl::ProjectPath& working,
-                const hecl::ProjectPath& cooked,
-                bool pcTarget);
 };
 
 }

Editor/ViewManager.cpp

@@ -303,6 +303,7 @@ bool ViewManager::proc()
     CGraphics::EndScene();
     gfxQ->execute();
     m_voiceEngine->pumpAndMixVoices();
+    m_projManager.asyncIdle();
     m_mainWindow->waitForRetrace();
 
     return true;
@@ -311,6 +312,7 @@ bool ViewManager::proc()
 void ViewManager::stop()
 {
     m_videoVoice.reset();
+    m_projManager.shutdown();
     CElementGen::Shutdown();
     CMoviePlayer::Shutdown();
     CLineRenderer::Shutdown();

hecl

@@ -1 +1 @@
-Subproject commit 50b614abd46fc59f2cd2ca0b32261263baffd2f2
+Subproject commit ed01d05677cb57c7d8cb69da85ebdaf7aed08e89