Refactorings to support .upak generation

DataSpec/DNAMP1/ANCS.cpp

@@ -1130,7 +1130,8 @@ bool ANCS::Cook(const hecl::ProjectPath& outPath,
                     if (sub.overlayMeshes.size())
                     {
                         ch.cmdlOverlay = sub.overlayMeshes[0].second;
-                        ch.cskrOverlay = inPath.ensureAuxInfo(chSysName.sys_str() + _S(".over.CSKR"));
+                        ch.cskrOverlay = inPath.ensureAuxInfo(chSysName.sys_str() + _S('.') +
+                                                                  sub.overlayMeshes[0].first + _S(".CSKR"));
                     }
 
                     break;
@@ -1206,7 +1207,15 @@ bool ANCS::CookCSKR(const hecl::ProjectPath& outPath,
 {
     hecl::SystemString subName(inPath.getAuxInfo().begin(),
                                inPath.getAuxInfo().end() - 5);
+    hecl::SystemString overName;
+    auto dotPos = subName.rfind(_S('.'));
+    if (dotPos != hecl::SystemString::npos)
+    {
+        overName = hecl::SystemString(subName.begin() + dotPos + 1, subName.end());
+        subName = hecl::SystemString(subName.begin(), subName.begin() + dotPos);
+    }
     hecl::SystemUTF8View subNameView(subName);
+    hecl::SystemUTF8View overNameView(overName);
 
     /* Build bone ID map */
     std::unordered_map<std::string, atInt32> boneIdMap;
@@ -1227,15 +1236,30 @@ bool ANCS::CookCSKR(const hecl::ProjectPath& outPath,
     if (!subtype)
         Log.report(logvisor::Fatal, _S("unable to find subtype '%s'"), subName.c_str());
 
-    const hecl::ProjectPath& modelPath = subtype->mesh;
+    const hecl::ProjectPath* modelPath = nullptr;
+    if (overName.empty())
+    {
+        modelPath = &subtype->mesh;
+    }
+    else
+    {
+        for (const auto& overlay : subtype->overlayMeshes)
+            if (!overlay.first.compare(overNameView.str()))
+            {
+                modelPath = &overlay.second;
+                break;
+            }
+    }
+    if (!modelPath)
+        Log.report(logvisor::Fatal, _S("unable to resolve model path of %s:%s"), subName.c_str(), overName.c_str());
 
-    if (!modelPath.isFile())
+    if (!modelPath->isFile())
-        Log.report(logvisor::Fatal, _S("unable to resolve '%s'"), modelPath.getRelativePath().c_str());
+        Log.report(logvisor::Fatal, _S("unable to resolve '%s'"), modelPath->getRelativePath().c_str());
 
-    hecl::ProjectPath skinIntPath = modelPath.getCookedPath(SpecEntMP1PC).getWithExtension(_S(".skinint"));
+    hecl::ProjectPath skinIntPath = modelPath->getCookedPath(SpecEntMP1PC).getWithExtension(_S(".skinint"));
-    if (!skinIntPath.isFileOrGlob() || skinIntPath.getModtime() < modelPath.getModtime())
+    if (!skinIntPath.isFileOrGlob() || skinIntPath.getModtime() < modelPath->getModtime())
-        if (!modelCookFunc(modelPath))
+        if (!modelCookFunc(*modelPath))
-            Log.report(logvisor::Fatal, _S("unable to cook '%s'"), modelPath.getRelativePath().c_str());
+            Log.report(logvisor::Fatal, _S("unable to cook '%s'"), modelPath->getRelativePath().c_str());
 
     athena::io::FileReader skinIO(skinIntPath.getAbsolutePath(), 1024*32, false);
     if (skinIO.hasError())

DataSpec/DNAMP1/DNAMP1.cpp

@@ -244,7 +244,7 @@ void PAKBridge::addCMDLRigPairs(PAKRouter<PAKBridge>& pakRouter,
                 if (ci.cmdlOverlay && ci.cskrOverlay)
                 {
                     addTo[ci.cmdlOverlay] = std::make_pair(ci.cskrOverlay, ci.cinf);
-                    cskrCinfToAncs[ci.cskrOverlay] = std::make_pair(entry.second.id, hecl::Format("%s.over.CSKR", ci.name.c_str()));
+                    cskrCinfToAncs[ci.cskrOverlay] = std::make_pair(entry.second.id, hecl::Format("%s.OVER.CSKR", ci.name.c_str()));
                     PAK::Entry* cmdlEnt = (PAK::Entry*)m_pak.lookupEntry(ci.cmdlOverlay);
                     PAK::Entry* cskrEnt = (PAK::Entry*)m_pak.lookupEntry(ci.cskrOverlay);
                     cmdlEnt->name = hecl::Format("ANCS_%08X_%s_overmodel", entry.first.toUint32(), ci.name.c_str());

DataSpec/DNAMP1/MLVL.cpp

@@ -73,9 +73,10 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
     hecl::ProjectPath namePath(inPath.getParentPath(), _S("!name.yaml"));
     if (namePath.isFile())
         mlvl.worldNameId = namePath;
-    hecl::ProjectPath savwPath = inPath.ensureAuxInfo(_S("SAVW"));
+    hecl::ProjectPath globPath = inPath.getWithExtension(_S(".*"), true);
+    hecl::ProjectPath savwPath = globPath.ensureAuxInfo(_S("SAVW"));
     mlvl.saveWorldId = savwPath;
-    hecl::ProjectPath mapwPath = inPath.ensureAuxInfo(_S("MAPW"));
+    hecl::ProjectPath mapwPath = globPath.ensureAuxInfo(_S("MAPW"));
     mlvl.worldMap = mapwPath;
 
     size_t areaIdx = 0;
@@ -269,7 +270,7 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
                     if (addedPaths.find(path.hash()) == addedPaths.cend())
                     {
                         addedPaths.insert(path.hash());
-                        urde::SObjectTag tag = g_curSpec->BuildTagFromPath(path, btok);
+                        urde::SObjectTag tag = g_curSpec->buildTagFromPath(path, btok);
                         if (tag.id.IsValid())
                             areaOut.deps.emplace_back(tag.id.Value(), tag.type);
                     }
@@ -308,13 +309,13 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
                 if (addedPaths.find(path.hash()) == addedPaths.cend())
                 {
                     addedPaths.insert(path.hash());
-                    urde::SObjectTag tag = g_curSpec->BuildTagFromPath(path, btok);
+                    urde::SObjectTag tag = g_curSpec->buildTagFromPath(path, btok);
                     if (tag.id.IsValid())
                         areaOut.deps.emplace_back(tag.id.Value(), tag.type);
                 }
             }
 
-            urde::SObjectTag tag = g_curSpec->BuildTagFromPath(areaPath, btok);
+            urde::SObjectTag tag = g_curSpec->buildTagFromPath(areaPath, btok);
             if (tag.id.IsValid())
                 areaOut.deps.emplace_back(tag.id.Value(), tag.type);
         }
@@ -359,7 +360,7 @@ bool MLVL::CookMAPW(const hecl::ProjectPath& outPath,
         /* Area map */
         hecl::ProjectPath mapPath(area.path, _S("/!map.blend"));
         if (mapPath.isFile())
-            mapaTags.push_back(g_curSpec->BuildTagFromPath(mapPath, btok));
+            mapaTags.push_back(g_curSpec->buildTagFromPath(mapPath, btok));
     }
 
     /* Write out MAPW */

DataSpec/DNAMP1/MREA.cpp

@@ -562,9 +562,9 @@ bool MREA::PCCook(const hecl::ProjectPath& outPath,
                 hecl::SNPrintf(thrIdx, 16, _S("%d"), hecl::ClientProcess::GetThreadWorkerIdx());
                 hecl::SystemChar parPid[32];
 #if _WIN32
-                hecl::SNPrintf(parPid, 32, _S("%ullX"), reinterpret_cast<unsigned long long>(GetCurrentProcess()));
+                hecl::SNPrintf(parPid, 32, _S("%lluX"), reinterpret_cast<unsigned long long>(GetCurrentProcess()));
 #else
-                hecl::SNPrintf(parPid, 32, _S("%ullX"), reinterpret_cast<unsigned long long>(getpid()));
+                hecl::SNPrintf(parPid, 32, _S("%lluX"), (unsigned long long)getpid());
 #endif
                 const hecl::SystemChar* args[] = {VisiGenPath.c_str(),
                                                   visiIntOut.getAbsolutePath().c_str(),

DataSpec/DNAMP2/DNAMP2.cpp

@@ -191,7 +191,7 @@ void PAKBridge::addCMDLRigPairs(PAKRouter<PAKBridge>& pakRouter,
                 if (ci.cmdlOverlay)
                 {
                     addTo[ci.cmdlOverlay] = std::make_pair(ci.cskrOverlay, ci.cinf);
-                    cskrCinfToAncs[ci.cskrOverlay] = std::make_pair(entry.second.id, hecl::Format("%s.over.CSKR", ci.name.c_str()));
+                    cskrCinfToAncs[ci.cskrOverlay] = std::make_pair(entry.second.id, hecl::Format("%s.OVER.CSKR", ci.name.c_str()));
                 }
             }
         }

DataSpec/SpecBase.cpp

@@ -5,14 +5,15 @@
 
 #include "SpecBase.hpp"
 #include "Blender/BlenderSupport.hpp"
-#include "hecl/Blender/BlenderConnection.hpp"
 #include "DNACommon/DNACommon.hpp"
 #include "DNACommon/TXTR.hpp"
 #include "AssetNameMap.hpp"
+#include "hecl/ClientProcess.hpp"
 
-#include <time.h>
 #include <png.h>
 
+#define DUMP_CACHE_FILL 1
+
 namespace DataSpec
 {
 
@@ -43,6 +44,11 @@ SpecBase::SpecBase(const hecl::Database::DataSpecEntry* specEntry, hecl::Databas
     DataSpec::UniqueIDBridge::setThreadProject(m_project);
 }
 
+SpecBase::~SpecBase()
+{
+    cancelBackgroundIndex();
+}
+
 static const hecl::SystemString regNONE = _S("");
 static const hecl::SystemString regE = _S("NTSC");
 static const hecl::SystemString regJ = _S("NTSC-J");
@@ -133,7 +139,15 @@ static bool IsPathAudioGroup(const hecl::ProjectPath& path)
         !path.getWithExtension(_S(".proj"), true).isFile() ||
         !path.getWithExtension(_S(".sdir"), true).isFile() ||
         !path.getWithExtension(_S(".samp"), true).isFile())
+    {
+        if (path.isFile() &&
+            !hecl::StrCmp(_S("proj"), path.getLastComponentExt()) &&
+            path.getWithExtension(_S(".pool"), true).isFile() &&
+            path.getWithExtension(_S(".sdir"), true).isFile() &&
+            path.getWithExtension(_S(".samp"), true).isFile())
+            return true;
         return false;
+    }
     return true;
 }
 
@@ -142,7 +156,13 @@ static bool IsPathSong(const hecl::ProjectPath& path)
     if (path.getPathType() != hecl::ProjectPath::Type::Glob ||
         !path.getWithExtension(_S(".mid"), true).isFile() ||
         !path.getWithExtension(_S(".yaml"), true).isFile())
+    {
+        if (path.isFile() &&
+            !hecl::StrCmp(_S("mid"), path.getLastComponentExt()) &&
+            path.getWithExtension(_S(".yaml"), true).isFile())
+            return true;
         return false;
+    }
     return true;
 }
 
@@ -359,10 +379,11 @@ void SpecBase::flattenDependencies(const hecl::ProjectPath& path,
                     const auto& arm = actor.armatures[sub.armature];
                     hecl::SystemStringView armSysName(arm.name);
                     pathsOut.push_back(path.ensureAuxInfo(armSysName.sys_str() + _S(".CINF")));
-                    if (sub.overlayMeshes.size())
+                    for (const auto& overlay : sub.overlayMeshes)
                     {
-                        pathsOut.push_back(sub.overlayMeshes[0].second);
+                        pathsOut.push_back(overlay.second);
-                        pathsOut.push_back(path.ensureAuxInfo(chSysName.sys_str() + _S(".over.CSKR")));
+                        pathsOut.push_back(path.ensureAuxInfo(chSysName.sys_str() + _S('.') +
+                                                                  overlay.first + _S(".CSKR")));
                     }
                 }
             }
@@ -402,13 +423,180 @@ void SpecBase::flattenDependencies(const UniqueID64& id, std::vector<hecl::Proje
         flattenDependencies(path, pathsOut, *g_ThreadBlenderToken.get());
 }
 
-bool SpecBase::canPackage(const PackagePassInfo& info)
+bool SpecBase::canPackage(const hecl::ProjectPath& path)
 {
+    auto components = path.getPathComponents();
+    if (components.size() <= 1)
+        return false;
+    hecl::ProjectPath outDir(m_project.getProjectWorkingPath(), _S("out"));
+
+    if (path.getPathType() == hecl::ProjectPath::Type::File &&
+        !hecl::StrCmp(path.getLastComponent(), _S("!world.blend")))
+        return true;
+    else if (path.getPathType() == hecl::ProjectPath::Type::Directory)
+        return true;
     return false;
 }
 
-void SpecBase::doPackage(const PackagePassInfo& info)
+void SpecBase::recursiveBuildResourceList(std::vector<urde::SObjectTag>& listOut,
+                                          const hecl::ProjectPath& path,
+                                          hecl::BlenderToken& btok)
 {
+    hecl::DirectoryEnumerator dEnum(path.getAbsolutePath(),
+                                    hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted, false, false, true);
+    for (const auto& ent : dEnum)
+    {
+        hecl::ProjectPath childPath(path, ent.m_name);
+        if (ent.m_isDir)
+            recursiveBuildResourceList(listOut, childPath, btok);
+        else if (urde::SObjectTag tag = tagFromPath(childPath, btok))
+            listOut.push_back(tag);
+    }
+}
+
+void SpecBase::copyBuildListData(std::vector<std::tuple<size_t, size_t, bool>>& fileIndex,
+                                 const std::vector<urde::SObjectTag>& buildList,
+                                 const hecl::Database::DataSpecEntry* entry,
+                                 bool fast, FProgress progress, athena::io::FileWriter& pakOut)
+{
+    fileIndex.reserve(buildList.size());
+    size_t loadIdx = 0;
+    for (const auto& tag : buildList)
+    {
+        fprintf(stderr, "\r %" PRISize " / %" PRISize "  %.4s %08X", ++loadIdx, buildList.size(),
+                tag.type.getChars(), (unsigned int)tag.id.Value());
+
+        fileIndex.emplace_back();
+        auto& thisIdx = fileIndex.back();
+        hecl::ProjectPath path = pathFromTag(tag);
+        hecl::ProjectPath cooked = getCookedPath(path, true);
+        athena::io::FileReader r(cooked.getAbsolutePath());
+        if (r.hasError())
+            Log.report(logvisor::Fatal, _S("Unable to open resource %s"), cooked.getRelativePath().c_str());
+        atUint64 size = r.length();
+        auto data = r.readUBytes(size);
+        auto compData = compressPakData(tag, data.get(), size);
+        if (compData.first)
+        {
+            std::get<0>(thisIdx) = pakOut.position();
+            std::get<1>(thisIdx) = ROUND_UP_32(compData.second + 4);
+            std::get<2>(thisIdx) = true;
+            pakOut.writeUint32Big(atUint32(size));
+            pakOut.writeUBytes(compData.first.get(), compData.second);
+            for (atUint64 i = compData.second + 4 ; i < std::get<1>(thisIdx) ; ++i)
+                pakOut.writeUByte(0xff);
+        }
+        else
+        {
+            std::get<0>(thisIdx) = pakOut.position();
+            std::get<1>(thisIdx) = ROUND_UP_32(size);
+            std::get<2>(thisIdx) = false;
+            pakOut.writeUBytes(data.get(), size);
+            for (atUint64 i = size ; i < std::get<1>(thisIdx) ; ++i)
+                pakOut.writeUByte(0xff);
+        }
+    }
+    fprintf(stderr, "\n");
+}
+
+void SpecBase::doPackage(const hecl::ProjectPath& path, const hecl::Database::DataSpecEntry* entry,
+                         bool fast, hecl::BlenderToken& btok, FProgress progress, hecl::ClientProcess* cp)
+{
+    /* Prepare complete resource index */
+    if (!m_backgroundRunning && m_tagToPath.empty())
+        beginBackgroundIndex();
+    waitForIndexComplete();
+
+    /* Name pak based on root-relative components */
+    auto components = path.getPathComponents();
+    if (components.size() <= 1)
+        return;
+    hecl::ProjectPath outPath(m_project.getProjectWorkingPath(),
+                             _S("out/") + components[0] + _S("/") + components[1] + _S(".upak"));
+    outPath.makeDirChain(false);
+    hecl::SystemString tmpPath = outPath.getAbsolutePath() + _S("~");
+
+    /* Output file */
+    athena::io::FileWriter pakOut(tmpPath);
+    std::vector<urde::SObjectTag> buildList;
+    atUint64 resTableOffset = 0;
+
+    if (path.getPathType() == hecl::ProjectPath::Type::File &&
+        !hecl::StrCmp(path.getLastComponent(), _S("!world.blend"))) /* World PAK */
+    {
+        /* Force-cook MLVL and write resource list structure */
+        m_project.cookPath(path, progress, false, true, fast);
+        hecl::ProjectPath cooked = getCookedPath(path, true);
+        buildWorldPakList(path, cooked, btok, pakOut, buildList, resTableOffset);
+        if (int64_t rem = pakOut.position() % 32)
+            for (int64_t i=0 ; i<32-rem ; ++i)
+                pakOut.writeUByte(0xff);
+    }
+    else if (path.getPathType() == hecl::ProjectPath::Type::Directory) /* General PAK */
+    {
+        /* Build resource list */
+        recursiveBuildResourceList(buildList, path, btok);
+        std::vector<std::pair<urde::SObjectTag, std::string>> nameList;
+
+        /* Build name list */
+        for (const auto& item : buildList)
+        {
+            auto search = m_catalogTagToName.find(item);
+            if (search != m_catalogTagToName.end())
+                nameList.emplace_back(item, search->second);
+        }
+
+        /* Write resource list structure */
+        buildPakList(btok, pakOut, buildList, nameList, resTableOffset);
+        if (int64_t rem = pakOut.position() % 32)
+            for (int64_t i=0 ; i<32-rem ; ++i)
+                pakOut.writeUByte(0xff);
+    }
+
+    /* Async cook resource list if using ClientProcess */
+    if (cp)
+    {
+        Log.report(logvisor::Info, _S("Validating resources"));
+        size_t loadIdx = 0;
+        for (auto& tag : buildList)
+        {
+            fprintf(stderr, "\r %" PRISize " / %" PRISize "  %.4s %08X", ++loadIdx, buildList.size(),
+                    tag.type.getChars(), (unsigned int)tag.id.Value());
+            hecl::ProjectPath depPath = pathFromTag(tag);
+            if (!depPath)
+            {
+                fprintf(stderr, "\n");
+                Log.report(logvisor::Fatal, _S("Unable to resolve %.4s %08X"),
+                           tag.type.getChars(), tag.id.Value());
+            }
+            m_project.cookPath(depPath, progress, false, false, fast, cp);
+        }
+        fprintf(stderr, "\n");
+        Log.report(logvisor::Info, _S("Waiting for remaining cook transactions"));
+        cp->waitUntilComplete();
+    }
+
+    /* Write resource data and build file index */
+    std::vector<std::tuple<size_t, size_t, bool>> fileIndex;
+    Log.report(logvisor::Info, _S("Copying data into %s"), outPath.getRelativePath().c_str());
+    copyBuildListData(fileIndex, buildList, entry, fast, progress, pakOut);
+
+    /* Write file index */
+    writePakFileIndex(pakOut, buildList, fileIndex, resTableOffset);
+    pakOut.close();
+
+    /* Atomic rename */
+    hecl::Rename(tmpPath.c_str(), outPath.getAbsolutePath().c_str());
+}
+
+hecl::ProjectPath SpecBase::getCookedPath(const hecl::ProjectPath& working, bool pcTarget) const
+{
+    const hecl::Database::DataSpecEntry* spec = &getOriginalSpec();
+    if (pcTarget)
+        spec = overrideDataSpec(working, getDataSpecEntry(), hecl::SharedBlenderToken);
+    if (!spec)
+        return {};
+    return working.getCookedPath(*spec);
 }
 
 static void PNGErr(png_structp png, png_const_charp msg)
@@ -427,7 +615,7 @@ static inline uint8_t Convert4To8(uint8_t v)
     return (v << 4) | v;
 }
 
-void SpecBase::ExtractRandomStaticEntropy(const uint8_t* buf, const hecl::ProjectPath& noAramPath)
+void SpecBase::extractRandomStaticEntropy(const uint8_t* buf, const hecl::ProjectPath& noAramPath)
 {
     hecl::ProjectPath entropyPath(noAramPath, _S("RandomStaticEntropy.png"));
     hecl::ProjectPath catalogPath(noAramPath, _S("!catalog.yaml"));
@@ -477,4 +665,570 @@ void SpecBase::ExtractRandomStaticEntropy(const uint8_t* buf, const hecl::Projec
     fclose(fp);
 }
 
+void SpecBase::clearTagCache()
+{
+    m_tagToPath.clear();
+    m_pathToTag.clear();
+    m_catalogNameToTag.clear();
+    m_catalogTagToName.clear();
+}
+
+hecl::ProjectPath SpecBase::pathFromTag(const urde::SObjectTag& tag) const
+{
+    std::unique_lock<std::mutex> lk(const_cast<SpecBase&>(*this).m_backgroundIndexMutex);
+    auto search = m_tagToPath.find(tag);
+    if (search != m_tagToPath.cend())
+        return search->second;
+    return {};
+}
+
+urde::SObjectTag SpecBase::tagFromPath(const hecl::ProjectPath& path,
+                                       hecl::BlenderToken& btok) const
+{
+    auto search = m_pathToTag.find(path.hash());
+    if (search != m_pathToTag.cend())
+        return search->second;
+    return buildTagFromPath(path, btok);
+}
+
+bool SpecBase::waitForTagReady(const urde::SObjectTag& tag, const hecl::ProjectPath*& pathOut)
+{
+    std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
+    auto search = m_tagToPath.find(tag);
+    if (search == m_tagToPath.end())
+    {
+        if (m_backgroundRunning)
+        {
+            while (m_backgroundRunning)
+            {
+                lk.unlock();
+                std::this_thread::sleep_for(std::chrono::milliseconds(2));
+                lk.lock();
+                search = m_tagToPath.find(tag);
+                if (search != m_tagToPath.end())
+                    break;
+            }
+            if (search == m_tagToPath.end())
+                return false;
+        }
+        else
+            return false;
+    }
+    lk.unlock();
+    pathOut = &search->second;
+    return true;
+}
+
+const urde::SObjectTag* SpecBase::getResourceIdByName(const char* name) const
+{
+    std::string lower = name;
+    std::transform(lower.cbegin(), lower.cend(), lower.begin(), tolower);
+
+    std::unique_lock<std::mutex> lk(const_cast<SpecBase&>(*this).m_backgroundIndexMutex);
+    auto search = m_catalogNameToTag.find(lower);
+    if (search == m_catalogNameToTag.end())
+    {
+        if (m_backgroundRunning)
+        {
+            while (m_backgroundRunning)
+            {
+                lk.unlock();
+                std::this_thread::sleep_for(std::chrono::milliseconds(2));
+                lk.lock();
+                search = m_catalogNameToTag.find(lower);
+                if (search != m_catalogNameToTag.end())
+                    break;
+            }
+            if (search == m_catalogNameToTag.end())
+                return nullptr;
+        }
+        else
+            return nullptr;
+    }
+    return &search->second;
+}
+
+FourCC SpecBase::getResourceTypeById(urde::CAssetId id) const
+{
+    if (!id.IsValid())
+        return {};
+
+    std::unique_lock<std::mutex> lk(const_cast<SpecBase&>(*this).m_backgroundIndexMutex);
+    urde::SObjectTag searchTag = {FourCC(), id};
+    auto search = m_tagToPath.find(searchTag);
+    if (search == m_tagToPath.end())
+    {
+        if (m_backgroundRunning)
+        {
+            while (m_backgroundRunning)
+            {
+                lk.unlock();
+                std::this_thread::sleep_for(std::chrono::milliseconds(2));
+                lk.lock();
+                search = m_tagToPath.find(searchTag);
+                if (search != m_tagToPath.end())
+                    break;
+            }
+            if (search == m_tagToPath.end())
+                return {};
+        }
+        else
+            return {};
+    }
+
+    return search->first.type;
+}
+
+void SpecBase::enumerateResources(const std::function<bool(const urde::SObjectTag&)>& lambda) const
+{
+    waitForIndexComplete();
+    for (const auto& pair : m_tagToPath)
+    {
+        if (!lambda(pair.first))
+        break;
+    }
+}
+
+void SpecBase::enumerateNamedResources(
+    const std::function<bool(const std::string&, const urde::SObjectTag&)>& lambda) const
+{
+    waitForIndexComplete();
+    for (const auto& pair : m_catalogNameToTag)
+    {
+        if (!lambda(pair.first, pair.second))
+            break;
+    }
+}
+
+static void WriteTag(athena::io::YAMLDocWriter& cacheWriter,
+                     const urde::SObjectTag& pathTag, const hecl::ProjectPath& path)
+{
+    char idStr[9];
+    snprintf(idStr, 9, "%08X", uint32_t(pathTag.id.Value()));
+    if (auto v = cacheWriter.enterSubVector(idStr))
+    {
+        cacheWriter.writeString(nullptr, pathTag.type.toString().c_str());
+        cacheWriter.writeString(nullptr, path.getAuxInfo().size() ?
+                                         (path.getRelativePathUTF8() + '|' + path.getAuxInfoUTF8()) :
+                                         path.getRelativePathUTF8());
+    }
+}
+
+static void WriteNameTag(athena::io::YAMLDocWriter& nameWriter,
+                         const urde::SObjectTag& pathTag,
+                         const std::string& name)
+{
+    char idStr[9];
+    snprintf(idStr, 9, "%08X", uint32_t(pathTag.id.Value()));
+    nameWriter.writeString(name.c_str(), idStr);
+}
+
+void SpecBase::readCatalog(const hecl::ProjectPath& catalogPath,
+                           athena::io::YAMLDocWriter& nameWriter)
+{
+    athena::io::FileReader freader(catalogPath.getAbsolutePath());
+    if (!freader.isOpen())
+        return;
+
+    athena::io::YAMLDocReader reader;
+    bool res = reader.parse(&freader);
+    if (!res)
+        return;
+
+    const athena::io::YAMLNode* root = reader.getRootNode();
+    for (const auto& p : root->m_mapChildren)
+    {
+        /* Hash as lowercase since lookup is case-insensitive */
+        std::string pLower = p.first;
+        std::transform(pLower.cbegin(), pLower.cend(), pLower.begin(), tolower);
+
+        /* Avoid redundant filesystem access for re-caches */
+        if (m_catalogNameToTag.find(pLower) != m_catalogNameToTag.cend())
+            continue;
+
+        athena::io::YAMLNode& node = *p.second;
+        hecl::ProjectPath path;
+        if (node.m_type == YAML_SCALAR_NODE)
+        {
+            path = hecl::ProjectPath(m_project.getProjectWorkingPath(), node.m_scalarString);
+        }
+        else if (node.m_type == YAML_SEQUENCE_NODE)
+        {
+            if (node.m_seqChildren.size() >= 2)
+                path = hecl::ProjectPath(m_project.getProjectWorkingPath(), node.m_seqChildren[0]->m_scalarString).
+                    ensureAuxInfo(node.m_seqChildren[1]->m_scalarString);
+            else if (node.m_seqChildren.size() == 1)
+                path = hecl::ProjectPath(m_project.getProjectWorkingPath(), node.m_seqChildren[0]->m_scalarString);
+        }
+        if (!path.isFileOrGlob())
+            continue;
+        urde::SObjectTag pathTag = tagFromPath(path, m_backgroundBlender);
+        if (pathTag)
+        {
+            std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
+            m_catalogNameToTag[pLower] = pathTag;
+            m_catalogTagToName[pathTag] = p.first;
+            WriteNameTag(nameWriter, pathTag, p.first);
+#if 0
+            fprintf(stderr, "%s %s %08X\n",
+                    p.first.c_str(),
+                    pathTag.type.toString().c_str(), uint32_t(pathTag.id));
+#endif
+        }
+    }
+}
+
+void SpecBase::backgroundIndexRecursiveCatalogs(const hecl::ProjectPath& dir,
+                                                athena::io::YAMLDocWriter& nameWriter,
+                                                int level)
+{
+    hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(),
+                                    hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted,
+                                    false, false, true);
+
+    /* Enumerate all items */
+    for (const hecl::DirectoryEnumerator::Entry& ent : dEnum)
+    {
+        hecl::ProjectPath path(dir, ent.m_name);
+        if (ent.m_isDir && level < 1)
+            backgroundIndexRecursiveCatalogs(path, nameWriter, level + 1);
+        else
+        {
+            if (!path.isFile())
+                continue;
+
+            /* Read catalog.yaml for .pak directory if exists */
+            if (level == 1 && !ent.m_name.compare(_S("!catalog.yaml")))
+            {
+                readCatalog(path, nameWriter);
+                continue;
+            }
+        }
+
+        /* bail if cancelled by client */
+        if (!m_backgroundRunning)
+            break;
+    }
+}
+
+#if DUMP_CACHE_FILL
+static void DumpCacheAdd(const urde::SObjectTag& pathTag, const hecl::ProjectPath& path)
+{
+    fprintf(stderr, "%s %08X %s\n",
+            pathTag.type.toString().c_str(), uint32_t(pathTag.id.Value()),
+            path.getRelativePathUTF8().c_str());
+}
+#endif
+
+bool SpecBase::addFileToIndex(const hecl::ProjectPath& path,
+                              athena::io::YAMLDocWriter& cacheWriter)
+{
+    /* Avoid redundant filesystem access for re-caches */
+    if (m_pathToTag.find(path.hash()) != m_pathToTag.cend())
+        return true;
+
+    /* Try as glob */
+    hecl::ProjectPath asGlob = path.getWithExtension(_S(".*"), true);
+    if (m_pathToTag.find(asGlob.hash()) != m_pathToTag.cend())
+        return true;
+
+    /* Classify intermediate into tag */
+    urde::SObjectTag pathTag = buildTagFromPath(path, m_backgroundBlender);
+    if (pathTag)
+    {
+        std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
+        bool useGlob = false;
+
+        /* Special multi-resource intermediates */
+        if (pathTag.type == SBIG('ANCS'))
+        {
+            hecl::BlenderConnection& conn = m_backgroundBlender.getBlenderConnection();
+            if (!conn.openBlend(path) || conn.getBlendType() != hecl::BlenderConnection::BlendType::Actor)
+                return false;
+
+            /* Transform tag to glob */
+            pathTag = {SBIG('ANCS'), asGlob.hash().val32()};
+            useGlob = true;
+
+            hecl::BlenderConnection::DataStream ds = conn.beginData();
+            std::vector<std::string> armatureNames = ds.getArmatureNames();
+            std::vector<std::string> subtypeNames = ds.getSubtypeNames();
+            std::vector<std::string> actionNames = ds.getActionNames();
+
+            for (const std::string& arm : armatureNames)
+            {
+                hecl::SystemStringView sysStr(arm);
+                hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".CINF"));
+                urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender);
+                m_tagToPath[pathTag] = subPath;
+                m_pathToTag[subPath.hash()] = pathTag;
+                WriteTag(cacheWriter, pathTag, subPath);
+#if DUMP_CACHE_FILL
+                DumpCacheAdd(pathTag, subPath);
+#endif
+            }
+
+            for (const std::string& sub : subtypeNames)
+            {
+                hecl::SystemStringView sysStr(sub);
+                hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".CSKR"));
+                urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender);
+                m_tagToPath[pathTag] = subPath;
+                m_pathToTag[subPath.hash()] = pathTag;
+                WriteTag(cacheWriter, pathTag, subPath);
+#if DUMP_CACHE_FILL
+                DumpCacheAdd(pathTag, subPath);
+#endif
+
+                std::vector<std::string> overlayNames = ds.getSubtypeOverlayNames(sub);
+                for (const auto& overlay : overlayNames)
+                {
+                    hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S('.') +
+                                                                         overlay + _S(".CSKR"));
+                    urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender);
+                    m_tagToPath[pathTag] = subPath;
+                    m_pathToTag[subPath.hash()] = pathTag;
+                    WriteTag(cacheWriter, pathTag, subPath);
+#if DUMP_CACHE_FILL
+                    DumpCacheAdd(pathTag, subPath);
+#endif
+                }
+            }
+
+            for (const std::string& act : actionNames)
+            {
+                hecl::SystemStringView sysStr(act);
+                hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".ANIM"));
+                urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender);
+                m_tagToPath[pathTag] = subPath;
+                m_pathToTag[subPath.hash()] = pathTag;
+                WriteTag(cacheWriter, pathTag, subPath);
+#if DUMP_CACHE_FILL
+                DumpCacheAdd(pathTag, subPath);
+#endif
+            }
+        }
+        else if (pathTag.type == SBIG('MLVL'))
+        {
+            /* Transform tag to glob */
+            pathTag = {SBIG('MLVL'), asGlob.hash().val32()};
+            useGlob = true;
+
+            hecl::ProjectPath subPath = asGlob.ensureAuxInfo(_S("MAPW"));
+            urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender);
+            m_tagToPath[pathTag] = subPath;
+            m_pathToTag[subPath.hash()] = pathTag;
+            WriteTag(cacheWriter, pathTag, subPath);
+#if DUMP_CACHE_FILL
+            DumpCacheAdd(pathTag, subPath);
+#endif
+
+            subPath = asGlob.ensureAuxInfo(_S("SAVW"));
+            pathTag = buildTagFromPath(subPath, m_backgroundBlender);
+            m_tagToPath[pathTag] = subPath;
+            m_pathToTag[subPath.hash()] = pathTag;
+            WriteTag(cacheWriter, pathTag, subPath);
+#if DUMP_CACHE_FILL
+            DumpCacheAdd(pathTag, subPath);
+#endif
+        }
+        else if (pathTag.type == SBIG('AGSC'))
+        {
+            /* Transform tag to glob */
+            pathTag = {SBIG('AGSC'), asGlob.hash().val32()};
+            useGlob = true;
+        }
+        else if (pathTag.type == SBIG('MREA'))
+        {
+            hecl::ProjectPath subPath = path.ensureAuxInfo(_S("PATH"));
+            urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender);
+            m_tagToPath[pathTag] = subPath;
+            m_pathToTag[subPath.hash()] = pathTag;
+            WriteTag(cacheWriter, pathTag, subPath);
+#if DUMP_CACHE_FILL
+            DumpCacheAdd(pathTag, subPath);
+#endif
+        }
+
+        /* Cache in-memory */
+        const hecl::ProjectPath& usePath = useGlob ? asGlob : path;
+        m_tagToPath[pathTag] = usePath;
+        m_pathToTag[usePath.hash()] = pathTag;
+        WriteTag(cacheWriter, pathTag, usePath);
+#if DUMP_CACHE_FILL
+        DumpCacheAdd(pathTag, usePath);
+#endif
+    }
+
+    return true;
+}
+
+void SpecBase::backgroundIndexRecursiveProc(const hecl::ProjectPath& dir,
+                                            athena::io::YAMLDocWriter& cacheWriter,
+                                            athena::io::YAMLDocWriter& nameWriter,
+                                            int level)
+{
+    hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(),
+                                    hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted,
+                                    false, false, true);
+
+    /* Enumerate all items */
+    for (const hecl::DirectoryEnumerator::Entry& ent : dEnum)
+    {
+        hecl::ProjectPath path(dir, ent.m_name);
+        if (ent.m_isDir)
+            backgroundIndexRecursiveProc(path, cacheWriter, nameWriter, level + 1);
+        else
+        {
+            if (!path.isFile())
+                continue;
+
+            /* Read catalog.yaml for .pak directory if exists */
+            if (level == 1 && !ent.m_name.compare(_S("!catalog.yaml")))
+            {
+                readCatalog(path, nameWriter);
+                continue;
+            }
+
+            /* Index the regular file */
+            addFileToIndex(path, cacheWriter);
+        }
+
+        /* bail if cancelled by client */
+        if (!m_backgroundRunning)
+            break;
+    }
+}
+
+void SpecBase::backgroundIndexProc()
+{
+    logvisor::RegisterThreadName("Resource Index Thread");
+
+    hecl::ProjectPath tagCachePath(m_project.getProjectCookedPath(getOriginalSpec()), _S("tag_cache.yaml"));
+    hecl::ProjectPath nameCachePath(m_project.getProjectCookedPath(getOriginalSpec()), _S("name_cache.yaml"));
+    hecl::ProjectPath specRoot(m_project.getProjectWorkingPath(), getOriginalSpec().m_name);
+
+    /* Cache will be overwritten with validated entries afterwards */
+    athena::io::YAMLDocWriter cacheWriter(nullptr);
+    athena::io::YAMLDocWriter nameWriter(nullptr);
+
+    /* Read in tag cache */
+    if (tagCachePath.isFile())
+    {
+        athena::io::FileReader reader(tagCachePath.getAbsolutePath());
+        if (reader.isOpen())
+        {
+            Log.report(logvisor::Info, _S("Cache index of '%s' loading"), getOriginalSpec().m_name);
+            athena::io::YAMLDocReader cacheReader;
+            if (cacheReader.parse(&reader))
+            {
+                std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
+                size_t tagCount = cacheReader.getRootNode()->m_mapChildren.size();
+                m_tagToPath.reserve(tagCount);
+                m_pathToTag.reserve(tagCount);
+                size_t loadIdx = 0;
+                for (const auto& child : cacheReader.getRootNode()->m_mapChildren)
+                {
+                    const athena::io::YAMLNode& node = *child.second;
+                    unsigned long id = strtoul(child.first.c_str(), nullptr, 16);
+                    hecl::FourCC type(node.m_seqChildren.at(0)->m_scalarString.c_str());
+                    hecl::ProjectPath path(m_project.getProjectWorkingPath(),
+                                           node.m_seqChildren.at(1)->m_scalarString);
+
+                    if (path.isFileOrGlob())
+                    {
+                        urde::SObjectTag pathTag(type, id);
+                        m_tagToPath[pathTag] = path;
+                        m_pathToTag[path.hash()] = pathTag;
+                        WriteTag(cacheWriter, pathTag, path);
+                    }
+                    fprintf(stderr, "\r %" PRISize " / %" PRISize, ++loadIdx, tagCount);
+                }
+                fprintf(stderr, "\n");
+            }
+            Log.report(logvisor::Info, _S("Cache index of '%s' loaded; %d tags"),
+                       getOriginalSpec().m_name, m_tagToPath.size());
+
+            if (nameCachePath.isFile())
+            {
+                /* Read in name cache */
+                Log.report(logvisor::Info, _S("Name index of '%s' loading"), getOriginalSpec().m_name);
+                athena::io::FileReader nreader(nameCachePath.getAbsolutePath());
+                athena::io::YAMLDocReader nameReader;
+                if (nameReader.parse(&nreader))
+                {
+                    std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
+                    m_catalogNameToTag.reserve(nameReader.getRootNode()->m_mapChildren.size());
+                    m_catalogTagToName.reserve(nameReader.getRootNode()->m_mapChildren.size());
+                    for (const auto& child : nameReader.getRootNode()->m_mapChildren)
+                    {
+                        unsigned long id = strtoul(child.second->m_scalarString.c_str(), nullptr, 16);
+                        auto search = m_tagToPath.find(urde::SObjectTag(FourCC(), uint32_t(id)));
+                        if (search != m_tagToPath.cend())
+                        {
+                            std::string chLower = child.first;
+                            std::transform(chLower.cbegin(), chLower.cend(), chLower.begin(), tolower);
+                            m_catalogNameToTag[chLower] = search->first;
+                            m_catalogTagToName[search->first] = child.first;
+                            WriteNameTag(nameWriter, search->first, child.first);
+                        }
+                    }
+                }
+                Log.report(logvisor::Info, _S("Name index of '%s' loaded; %d names"),
+                           getOriginalSpec().m_name, m_catalogNameToTag.size());
+            }
+        }
+    }
+
+    /* Add special original IDs resource if exists (not name-cached to disk) */
+    hecl::ProjectPath oidsPath(specRoot, "!original_ids.yaml");
+    urde::SObjectTag oidsTag = buildTagFromPath(oidsPath, m_backgroundBlender);
+    if (oidsTag)
+    {
+        m_catalogNameToTag["mp1originalids"] = oidsTag;
+        m_catalogTagToName[oidsTag] = "MP1OriginalIDs";
+    }
+
+    Log.report(logvisor::Info, _S("Background index of '%s' started"), getOriginalSpec().m_name);
+    backgroundIndexRecursiveProc(specRoot, cacheWriter, nameWriter, 0);
+
+    tagCachePath.makeDirChain(false);
+    athena::io::FileWriter twriter(tagCachePath.getAbsolutePath());
+    cacheWriter.finish(&twriter);
+
+    athena::io::FileWriter nwriter(nameCachePath.getAbsolutePath());
+    nameWriter.finish(&nwriter);
+
+    m_backgroundBlender.shutdown();
+    Log.report(logvisor::Info, _S("Background index of '%s' complete; %d tags, %d names"),
+               getOriginalSpec().m_name, m_tagToPath.size(), m_catalogNameToTag.size());
+    m_backgroundRunning = false;
+}
+
+void SpecBase::cancelBackgroundIndex()
+{
+    m_backgroundRunning = false;
+    if (m_backgroundIndexTh.joinable())
+        m_backgroundIndexTh.join();
+}
+
+void SpecBase::beginBackgroundIndex()
+{
+    cancelBackgroundIndex();
+    clearTagCache();
+    m_backgroundRunning = true;
+    m_backgroundIndexTh = std::thread(std::bind(&SpecBase::backgroundIndexProc, this));
+}
+
+void SpecBase::waitForIndexComplete() const
+{
+    std::unique_lock<std::mutex> lk(const_cast<SpecBase&>(*this).m_backgroundIndexMutex);
+    while (m_backgroundRunning)
+    {
+        lk.unlock();
+        std::this_thread::sleep_for(std::chrono::milliseconds(2));
+        lk.lock();
+    }
+}
+
 }

DataSpec/SpecBase.hpp

@@ -2,15 +2,14 @@
 #define SPECBASE_HPP
 
 #include <functional>
+#include <thread>
 
 #include <hecl/Database.hpp>
 #include <nod/nod.hpp>
+#include <athena/DNAYaml.hpp>
+#include <athena/FileWriter.hpp>
 #include "hecl/Blender/BlenderConnection.hpp"
-
+#include "Runtime/RetroTypes.hpp"
-namespace urde
-{
-struct SObjectTag;
-}
 
 namespace DataSpec
 {
@@ -29,8 +28,9 @@ struct SpecBase : hecl::Database::IDataSpec
     void doCook(const hecl::ProjectPath& path, const hecl::ProjectPath& cookedPath,
                 bool fast, hecl::BlenderToken& btok, FCookProgress progress);
 
-    bool canPackage(const PackagePassInfo& info);
+    bool canPackage(const hecl::ProjectPath& path);
-    void doPackage(const PackagePassInfo& info);
+    void doPackage(const hecl::ProjectPath& path, const hecl::Database::DataSpecEntry* entry,
+                   bool fast, hecl::BlenderToken& btok, FProgress progress, hecl::ClientProcess* cp);
 
     /* Extract handlers */
     virtual bool checkStandaloneID(const char* id) const=0;
@@ -46,7 +46,7 @@ struct SpecBase : hecl::Database::IDataSpec
                                  FProgress progress)=0;
 
     /* Convert path to object tag */
-    virtual urde::SObjectTag BuildTagFromPath(const hecl::ProjectPath& path,
+    virtual urde::SObjectTag buildTagFromPath(const hecl::ProjectPath& path,
                                               hecl::BlenderToken& btok) const=0;
 
     /* Even if PC spec is being cooked, this will return the vanilla GCN spec */
@@ -107,23 +107,93 @@ struct SpecBase : hecl::Database::IDataSpec
     void flattenDependencies(const class UniqueID64& id, std::vector<hecl::ProjectPath>& pathsOut);
     virtual void flattenDependenciesYAML(athena::io::IStreamReader& fin, std::vector<hecl::ProjectPath>& pathsOut)=0;
 
+    virtual void buildWorldPakList(const hecl::ProjectPath& worldPath,
+                                   const hecl::ProjectPath& worldPathCooked,
+                                   hecl::BlenderToken& btok,
+                                   athena::io::FileWriter& w,
+                                   std::vector<urde::SObjectTag>& listOut,
+                                   atUint64& resTableOffset) {}
+    virtual void buildPakList(hecl::BlenderToken& btok,
+                              athena::io::FileWriter& w,
+                              const std::vector<urde::SObjectTag>& list,
+                              const std::vector<std::pair<urde::SObjectTag, std::string>>& nameList,
+                              atUint64& resTableOffset) {}
+    virtual void writePakFileIndex(athena::io::FileWriter& w,
+                                   const std::vector<urde::SObjectTag>& tags,
+                                   const std::vector<std::tuple<size_t, size_t, bool>>& index,
+                                   atUint64 resTableOffset) {}
+    virtual std::pair<std::unique_ptr<uint8_t[]>, size_t>
+    compressPakData(const urde::SObjectTag& tag, const uint8_t* data, size_t len) { return {}; }
+
     const hecl::ProjectPath& getMasterShaderPath() const {return m_masterShader;}
 
     /* Support functions for resolving paths from IDs */
     virtual hecl::ProjectPath getWorking(class UniqueID32&) {return hecl::ProjectPath();}
     virtual hecl::ProjectPath getWorking(class UniqueID64&) {return hecl::ProjectPath();}
 
+    hecl::ProjectPath getCookedPath(const hecl::ProjectPath& working, bool pcTarget) const;
+
     /* Project accessor */
     hecl::Database::Project& getProject() const {return m_project;}
 
     /* Extract RandomStatic entropy */
-    void ExtractRandomStaticEntropy(const uint8_t* buf, const hecl::ProjectPath& noAramPath);
+    void extractRandomStaticEntropy(const uint8_t* buf, const hecl::ProjectPath& noAramPath);
+
+    /* Tag cache functions */
+    urde::SObjectTag tagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const;
+    hecl::ProjectPath pathFromTag(const urde::SObjectTag& tag) const;
+    bool waitForTagReady(const urde::SObjectTag& tag, const hecl::ProjectPath*& pathOut);
+    const urde::SObjectTag* getResourceIdByName(const char* name) const;
+    hecl::FourCC getResourceTypeById(urde::CAssetId id) const;
+    void enumerateResources(const std::function<bool(const urde::SObjectTag&)>& lambda) const;
+    void enumerateNamedResources(
+        const std::function<bool(const std::string&, const urde::SObjectTag&)>& lambda) const;
+    void cancelBackgroundIndex();
+    void beginBackgroundIndex();
+    bool backgroundIndexRunning() const { return m_backgroundRunning; }
+    void waitForIndexComplete() const;
+
+    virtual void getTagListForFile(const char* pakName, std::vector<urde::SObjectTag>& out) const {}
 
     SpecBase(const hecl::Database::DataSpecEntry* specEntry, hecl::Database::Project& project, bool pc);
+    ~SpecBase();
 protected:
     hecl::Database::Project& m_project;
     bool m_pc;
     hecl::ProjectPath m_masterShader;
+
+    std::unordered_map<urde::SObjectTag, hecl::ProjectPath> m_tagToPath;
+    std::unordered_map<hecl::Hash, urde::SObjectTag> m_pathToTag;
+    std::unordered_map<std::string, urde::SObjectTag> m_catalogNameToTag;
+    std::unordered_map<urde::SObjectTag, std::string> m_catalogTagToName;
+    void clearTagCache();
+
+    hecl::BlenderToken m_backgroundBlender;
+    std::thread m_backgroundIndexTh;
+    std::mutex m_backgroundIndexMutex;
+    bool m_backgroundRunning = false;
+
+    void readCatalog(const hecl::ProjectPath& catalogPath,
+                     athena::io::YAMLDocWriter& nameWriter);
+    bool addFileToIndex(const hecl::ProjectPath& path,
+                        athena::io::YAMLDocWriter& cacheWriter);
+    void backgroundIndexRecursiveProc(const hecl::ProjectPath& path,
+                                      athena::io::YAMLDocWriter& cacheWriter,
+                                      athena::io::YAMLDocWriter& nameWriter,
+                                      int level);
+    void backgroundIndexRecursiveCatalogs(const hecl::ProjectPath& path,
+                                          athena::io::YAMLDocWriter& nameWriter,
+                                          int level);
+    void backgroundIndexProc();
+
+    void recursiveBuildResourceList(std::vector<urde::SObjectTag>& listOut,
+                                    const hecl::ProjectPath& path,
+                                    hecl::BlenderToken& btok);
+    void copyBuildListData(std::vector<std::tuple<size_t, size_t, bool>>& fileIndex,
+                           const std::vector<urde::SObjectTag>& buildList,
+                           const hecl::Database::DataSpecEntry* entry,
+                           bool fast, FProgress progress, athena::io::FileWriter& pakOut);
+
 private:
     std::unique_ptr<nod::DiscBase> m_disc;
     bool m_isWii;

DataSpec/SpecMP1.cpp

@@ -430,7 +430,7 @@ struct SpecMP1 : SpecBase
 
         /* Extract part of .dol for RandomStatic entropy */
         hecl::ProjectPath noAramPath(m_project.getProjectWorkingPath(), _S("MP1/NoARAM"));
-        ExtractRandomStaticEntropy(m_dolBuf.get() + 0x4f60, noAramPath);
+        extractRandomStaticEntropy(m_dolBuf.get() + 0x4f60, noAramPath);
 
         return true;
     }
@@ -517,7 +517,7 @@ struct SpecMP1 : SpecBase
         });
     }
 
-    urde::SObjectTag BuildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
+    urde::SObjectTag buildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
     {
         if (hecl::StringUtils::EndsWith(path.getAuxInfo(), _S(".CINF")))
             return {SBIG('CINF'), path.hash().val32()};
@@ -718,6 +718,18 @@ struct SpecMP1 : SpecBase
         return {};
     }
 
+    void getTagListForFile(const char* pakName, std::vector<urde::SObjectTag>& out) const
+    {
+        std::string pathPrefix("MP1/");
+        pathPrefix += pakName;
+        pathPrefix += '/';
+
+        std::unique_lock<std::mutex> lk(const_cast<SpecMP1&>(*this).m_backgroundIndexMutex);
+        for (const auto& tag : m_tagToPath)
+            if (!tag.second.getRelativePathUTF8().compare(0, pathPrefix.size(), pathPrefix))
+                out.push_back(tag.first);
+    }
+
     void cookMesh(const hecl::ProjectPath& out, const hecl::ProjectPath& in, BlendStream& ds, bool fast,
                   hecl::BlenderToken& btok, FCookProgress progress)
     {
@@ -1109,6 +1121,210 @@ struct SpecMP1 : SpecBase
         }
     }
 
+    void buildWorldPakList(const hecl::ProjectPath& worldPath,
+                           const hecl::ProjectPath& worldPathCooked,
+                           hecl::BlenderToken& btok,
+                           athena::io::FileWriter& w,
+                           std::vector<urde::SObjectTag>& listOut,
+                           atUint64& resTableOffset)
+    {
+        DNAMP1::MLVL mlvl;
+        {
+            athena::io::FileReader r(worldPathCooked.getAbsolutePath());
+            if (r.hasError())
+                Log.report(logvisor::Fatal, _S("Unable to open world %s"), worldPathCooked.getRelativePath().c_str());
+            mlvl.read(r);
+        }
+
+        size_t count = 5;
+        for (const auto& area : mlvl.areas)
+            for (const auto& dep : area.deps)
+                ++count;
+        listOut.reserve(count);
+
+        urde::SObjectTag worldTag = tagFromPath(worldPath.getWithExtension(_S(".*"), true), btok);
+
+        w.writeUint32Big(0x80030005);
+        w.writeUint32Big(0);
+
+        w.writeUint32Big(1);
+        DNAMP1::PAK::NameEntry nameEnt;
+        hecl::ProjectPath parentDir = worldPath.getParentPath();
+        nameEnt.type = worldTag.type;
+        nameEnt.id = atUint32(worldTag.id.Value());
+        nameEnt.nameLen = atUint32(hecl::StrLen(parentDir.getLastComponent()));
+        nameEnt.name = parentDir.getLastComponent();
+        nameEnt.write(w);
+
+        w.writeUint32Big(atUint32(count));
+        resTableOffset = w.position();
+        for (const auto& area : mlvl.areas)
+            for (const auto& dep : area.deps)
+                listOut.push_back({dep.type, dep.id.toUint32()});
+
+        urde::SObjectTag nameTag(FOURCC('STRG'), mlvl.worldNameId.toUint32());
+        if (nameTag)
+            listOut.push_back(nameTag);
+
+        urde::SObjectTag savwTag(FOURCC('SAVW'), mlvl.saveWorldId.toUint32());
+        if (savwTag)
+        {
+            if (hecl::ProjectPath savwPath = pathFromTag(savwTag))
+                m_project.cookPath(savwPath, {}, false, true);
+            listOut.push_back(savwTag);
+        }
+
+        urde::SObjectTag mapTag(FOURCC('MAPW'), mlvl.worldMap.toUint32());
+        if (mapTag)
+        {
+            if (hecl::ProjectPath mapPath = pathFromTag(mapTag))
+            {
+                m_project.cookPath(mapPath, {}, false, true);
+                if (hecl::ProjectPath mapCookedPath = getCookedPath(mapPath, true))
+                {
+                    athena::io::FileReader r(mapCookedPath.getAbsolutePath());
+                    if (r.hasError())
+                        Log.report(logvisor::Fatal, _S("Unable to open %s"), mapCookedPath.getRelativePath().c_str());
+
+                    if (r.readUint32Big() != 0xDEADF00D)
+                        Log.report(logvisor::Fatal, _S("Corrupt MAPW %s"), mapCookedPath.getRelativePath().c_str());
+                    r.readUint32Big();
+                    atUint32 mapaCount = r.readUint32Big();
+                    for (int i=0 ; i<mapaCount ; ++i)
+                    {
+                        UniqueID32 id;
+                        id.read(r);
+                        listOut.push_back({FOURCC('MAPA'), id.toUint32()});
+                    }
+                }
+            }
+            listOut.push_back(mapTag);
+        }
+
+        urde::SObjectTag skyboxTag(FOURCC('CMDL'), mlvl.worldSkyboxId.toUint32());
+        if (skyboxTag)
+        {
+            hecl::ProjectPath skyboxPath = pathFromTag(skyboxTag);
+            if (btok.getBlenderConnection().openBlend(skyboxPath))
+            {
+                auto data = btok.getBlenderConnection().beginData();
+                std::vector<hecl::ProjectPath> textures = data.getTextures();
+                for (const auto& tex : textures)
+                {
+                    urde::SObjectTag texTag = tagFromPath(tex, btok);
+                    if (!texTag)
+                        Log.report(logvisor::Fatal, _S("Unable to resolve %s"), tex.getRelativePath().c_str());
+                    listOut.push_back(texTag);
+                }
+            }
+            listOut.push_back(skyboxTag);
+        }
+
+        listOut.push_back(worldTag);
+
+        for (const auto& item : listOut)
+        {
+            DNAMP1::PAK::Entry ent;
+            ent.compressed = 0;
+            ent.type = item.type;
+            ent.id = atUint32(item.id.Value());
+            ent.size = 0;
+            ent.offset = 0;
+            ent.write(w);
+        }
+    }
+
+    void buildPakList(hecl::BlenderToken& btok,
+                      athena::io::FileWriter& w,
+                      const std::vector<urde::SObjectTag>& list,
+                      const std::vector<std::pair<urde::SObjectTag, std::string>>& nameList,
+                      atUint64& resTableOffset)
+    {
+        w.writeUint32Big(0x80030005);
+        w.writeUint32Big(0);
+
+        w.writeUint32Big(atUint32(nameList.size()));
+        for (const auto& item : nameList)
+        {
+            DNAMP1::PAK::NameEntry nameEnt;
+            nameEnt.type = item.first.type;
+            nameEnt.id = atUint32(item.first.id.Value());
+            nameEnt.nameLen = atUint32(item.second.size());
+            nameEnt.name = item.second;
+            nameEnt.write(w);
+        }
+
+        w.writeUint32Big(atUint32(list.size()));
+        resTableOffset = w.position();
+        for (const auto& item : list)
+        {
+            DNAMP1::PAK::Entry ent;
+            ent.compressed = 0;
+            ent.type = item.type;
+            ent.id = atUint32(item.id.Value());
+            ent.size = 0;
+            ent.offset = 0;
+            ent.write(w);
+        }
+    }
+
+    void writePakFileIndex(athena::io::FileWriter& w,
+                           const std::vector<urde::SObjectTag>& tags,
+                           const std::vector<std::tuple<size_t, size_t, bool>>& index,
+                           atUint64 resTableOffset)
+    {
+        w.seek(resTableOffset, athena::Begin);
+
+        auto it = tags.begin();
+        for (const auto& item : index)
+        {
+            const urde::SObjectTag& tag = *it++;
+            DNAMP1::PAK::Entry ent;
+            ent.compressed = atUint32(std::get<2>(item));
+            ent.type = tag.type;
+            ent.id = atUint32(tag.id.Value());
+            ent.size = atUint32(std::get<1>(item));
+            ent.offset = atUint32(std::get<0>(item));
+            ent.write(w);
+        }
+    }
+
+    std::pair<std::unique_ptr<uint8_t[]>, size_t>
+    compressPakData(const urde::SObjectTag& tag, const uint8_t* data, size_t len)
+    {
+        bool doCompress = false;
+        switch (tag.type)
+        {
+        case SBIG('TXTR'):
+        case SBIG('CMDL'):
+        case SBIG('CSKR'):
+        case SBIG('ANCS'):
+        case SBIG('ANIM'):
+        case SBIG('FONT'):
+            doCompress = true;
+            break;
+        case SBIG('PART'):
+        case SBIG('ELSC'):
+        case SBIG('SWHC'):
+        case SBIG('WPSC'):
+        case SBIG('DPSC'):
+        case SBIG('CRSC'):
+            doCompress = len >= 0x400;
+            break;
+        default:
+            break;
+        }
+        if (!doCompress)
+            return {};
+
+        uLong destLen = compressBound(len);
+        std::pair<std::unique_ptr<uint8_t[]>, size_t> ret;
+        ret.first.reset(new uint8_t[destLen]);
+        compress(ret.first.get(), &destLen, data, len);
+        ret.second = destLen;
+        return ret;
+    };
+
     void cookAudioGroup(const hecl::ProjectPath& out, const hecl::ProjectPath& in, FCookProgress progress)
     {
         DNAMP1::AGSC::Cook(in, out);

DataSpec/SpecMP2.cpp

@@ -318,7 +318,7 @@ struct SpecMP2 : SpecBase
         });
     }
 
-    urde::SObjectTag BuildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
+    urde::SObjectTag buildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
     {
         return {};
     }

DataSpec/SpecMP3.cpp

@@ -512,7 +512,7 @@ struct SpecMP3 : SpecBase
         return false;
     }
 
-    urde::SObjectTag BuildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
+    urde::SObjectTag buildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
     {
         return {};
     }

Editor/ProjectResourceFactoryBase.cpp

@@ -1,445 +1,21 @@
 #include "ProjectResourceFactoryBase.hpp"
 #include "Runtime/IObj.hpp"
 
-#define DUMP_CACHE_FILL 1
-
 namespace urde
 {
 static logvisor::Module Log("urde::ProjectResourceFactoryBase");
 
-static void WriteTag(athena::io::YAMLDocWriter& cacheWriter,
-                     const SObjectTag& pathTag, const hecl::ProjectPath& path)
-{
-    char idStr[9];
-    snprintf(idStr, 9, "%08X", uint32_t(pathTag.id.Value()));
-    if (auto v = cacheWriter.enterSubVector(idStr))
-    {
-        cacheWriter.writeString(nullptr, pathTag.type.toString().c_str());
-        cacheWriter.writeString(nullptr, path.getAuxInfo().size() ?
-            (path.getRelativePathUTF8() + '|' + path.getAuxInfoUTF8()) :
-             path.getRelativePathUTF8());
-    }
-}
-
-static void WriteNameTag(athena::io::YAMLDocWriter& nameWriter,
-                         const SObjectTag& pathTag,
-                         const std::string& name)
-{
-    char idStr[9];
-    snprintf(idStr, 9, "%08X", uint32_t(pathTag.id.Value()));
-    nameWriter.writeString(name.c_str(), idStr);
-}
-
-void ProjectResourceFactoryBase::Clear()
-{
-    m_tagToPath.clear();
-    m_pathToTag.clear();
-    m_catalogNameToTag.clear();
-}
-
-SObjectTag ProjectResourceFactoryBase::TagFromPath(const hecl::ProjectPath& path,
-                                                   hecl::BlenderToken& btok) const
-{
-    auto search = m_pathToTag.find(path.hash());
-    if (search != m_pathToTag.cend())
-        return search->second;
-    return BuildTagFromPath(path, btok);
-}
-
-void ProjectResourceFactoryBase::ReadCatalog(const hecl::ProjectPath& catalogPath,
-                                             athena::io::YAMLDocWriter& nameWriter)
-{
-    athena::io::FileReader freader(catalogPath.getAbsolutePath());
-    if (!freader.isOpen())
-        return;
-
-    athena::io::YAMLDocReader reader;
-    bool res = reader.parse(&freader);
-    if (!res)
-        return;
-
-    const athena::io::YAMLNode* root = reader.getRootNode();
-    for (const auto& p : root->m_mapChildren)
-    {
-        /* Hash as lowercase since lookup is case-insensitive */
-        std::string pLower = p.first;
-        std::transform(pLower.cbegin(), pLower.cend(), pLower.begin(), tolower);
-
-        /* Avoid redundant filesystem access for re-caches */
-        if (m_catalogNameToTag.find(pLower) != m_catalogNameToTag.cend())
-            continue;
-
-        athena::io::YAMLNode& node = *p.second;
-        hecl::ProjectPath path;
-        if (node.m_type == YAML_SCALAR_NODE)
-        {
-            path = hecl::ProjectPath(m_proj->getProjectWorkingPath(), node.m_scalarString);
-        }
-        else if (node.m_type == YAML_SEQUENCE_NODE)
-        {
-            if (node.m_seqChildren.size() >= 2)
-                path = hecl::ProjectPath(m_proj->getProjectWorkingPath(), node.m_seqChildren[0]->m_scalarString).
-                        ensureAuxInfo(node.m_seqChildren[1]->m_scalarString);
-            else if (node.m_seqChildren.size() == 1)
-                path = hecl::ProjectPath(m_proj->getProjectWorkingPath(), node.m_seqChildren[0]->m_scalarString);
-        }
-        if (!path.isFileOrGlob())
-            continue;
-        SObjectTag pathTag = TagFromPath(path, m_backgroundBlender);
-        if (pathTag)
-        {
-            std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
-            m_catalogNameToTag[pLower] = pathTag;
-            WriteNameTag(nameWriter, pathTag, p.first);
-#if 0
-            fprintf(stderr, "%s %s %08X\n",
-                    p.first.c_str(),
-                    pathTag.type.toString().c_str(), uint32_t(pathTag.id));
-#endif
-        }
-    }
-}
-
-void ProjectResourceFactoryBase::BackgroundIndexRecursiveCatalogs(const hecl::ProjectPath& dir,
-                                                                  athena::io::YAMLDocWriter& nameWriter,
-                                                                  int level)
-{
-    hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(),
-                                    hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted,
-                                    false, false, true);
-
-    /* Enumerate all items */
-    for (const hecl::DirectoryEnumerator::Entry& ent : dEnum)
-    {
-        hecl::ProjectPath path(dir, ent.m_name);
-        if (ent.m_isDir && level < 1)
-            BackgroundIndexRecursiveCatalogs(path, nameWriter, level+1);
-        else
-        {
-            if (!path.isFile())
-                continue;
-
-            /* Read catalog.yaml for .pak directory if exists */
-            if (level == 1 && !ent.m_name.compare(_S("!catalog.yaml")))
-            {
-                ReadCatalog(path, nameWriter);
-                continue;
-            }
-        }
-
-        /* bail if cancelled by client */
-        if (!m_backgroundRunning)
-            break;
-    }
-}
-
-#if DUMP_CACHE_FILL
-static void DumpCacheAdd(const SObjectTag& pathTag, const hecl::ProjectPath& path)
-{
-    fprintf(stderr, "%s %08X %s\n",
-            pathTag.type.toString().c_str(), uint32_t(pathTag.id.Value()),
-            path.getRelativePathUTF8().c_str());
-}
-#endif
-
-bool ProjectResourceFactoryBase::AddFileToIndex(const hecl::ProjectPath& path,
-                                                athena::io::YAMLDocWriter& cacheWriter)
-{
-    /* Avoid redundant filesystem access for re-caches */
-    if (m_pathToTag.find(path.hash()) != m_pathToTag.cend())
-        return true;
-
-    /* Try as glob */
-    hecl::ProjectPath asGlob = path.getWithExtension(_S(".*"), true);
-    if (m_pathToTag.find(asGlob.hash()) != m_pathToTag.cend())
-        return true;
-
-    /* Classify intermediate into tag */
-    SObjectTag pathTag = BuildTagFromPath(path, m_backgroundBlender);
-    if (pathTag)
-    {
-        std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
-        bool useGlob = false;
-
-        /* Special multi-resource intermediates */
-        if (pathTag.type == SBIG('ANCS'))
-        {
-            hecl::BlenderConnection& conn = m_backgroundBlender.getBlenderConnection();
-            if (!conn.openBlend(path) || conn.getBlendType() != hecl::BlenderConnection::BlendType::Actor)
-                return false;
-
-            /* Transform tag to glob */
-            pathTag = {SBIG('ANCS'), asGlob.hash().val32()};
-            useGlob = true;
-
-            hecl::BlenderConnection::DataStream ds = conn.beginData();
-            std::vector<std::string> armatureNames = ds.getArmatureNames();
-            std::vector<std::string> subtypeNames = ds.getSubtypeNames();
-            std::vector<std::string> actionNames = ds.getActionNames();
-
-            for (const std::string& arm : armatureNames)
-            {
-                hecl::SystemStringView sysStr(arm);
-                hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".CINF"));
-                SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
-                m_tagToPath[pathTag] = subPath;
-                m_pathToTag[subPath.hash()] = pathTag;
-                WriteTag(cacheWriter, pathTag, subPath);
-#if DUMP_CACHE_FILL
-                DumpCacheAdd(pathTag, subPath);
-#endif
-            }
-
-            for (const std::string& sub : subtypeNames)
-            {
-                hecl::SystemStringView sysStr(sub);
-                hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".CSKR"));
-                SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
-                m_tagToPath[pathTag] = subPath;
-                m_pathToTag[subPath.hash()] = pathTag;
-                WriteTag(cacheWriter, pathTag, subPath);
-#if DUMP_CACHE_FILL
-                DumpCacheAdd(pathTag, subPath);
-#endif
-            }
-
-            for (const std::string& act : actionNames)
-            {
-                hecl::SystemStringView sysStr(act);
-                hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".ANIM"));
-                SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
-                m_tagToPath[pathTag] = subPath;
-                m_pathToTag[subPath.hash()] = pathTag;
-                WriteTag(cacheWriter, pathTag, subPath);
-#if DUMP_CACHE_FILL
-                DumpCacheAdd(pathTag, subPath);
-#endif
-            }
-        }
-        else if (pathTag.type == SBIG('MLVL'))
-        {
-            /* Transform tag to glob */
-            pathTag = {SBIG('MLVL'), asGlob.hash().val32()};
-            useGlob = true;
-
-            hecl::ProjectPath subPath = asGlob.ensureAuxInfo(_S("MAPW"));
-            SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
-            m_tagToPath[pathTag] = subPath;
-            m_pathToTag[subPath.hash()] = pathTag;
-            WriteTag(cacheWriter, pathTag, subPath);
-#if DUMP_CACHE_FILL
-            DumpCacheAdd(pathTag, subPath);
-#endif
-
-            subPath = asGlob.ensureAuxInfo(_S("SAVW"));
-            pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
-            m_tagToPath[pathTag] = subPath;
-            m_pathToTag[subPath.hash()] = pathTag;
-            WriteTag(cacheWriter, pathTag, subPath);
-#if DUMP_CACHE_FILL
-            DumpCacheAdd(pathTag, subPath);
-#endif
-        }
-        else if (pathTag.type == SBIG('AGSC'))
-        {
-            /* Transform tag to glob */
-            pathTag = {SBIG('AGSC'), asGlob.hash().val32()};
-            useGlob = true;
-        }
-        else if (pathTag.type == SBIG('MREA'))
-        {
-            hecl::ProjectPath subPath = path.ensureAuxInfo(_S("PATH"));
-            SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
-            m_tagToPath[pathTag] = subPath;
-            m_pathToTag[subPath.hash()] = pathTag;
-            WriteTag(cacheWriter, pathTag, subPath);
-#if DUMP_CACHE_FILL
-            DumpCacheAdd(pathTag, subPath);
-#endif
-        }
-
-        /* Cache in-memory */
-        const hecl::ProjectPath& usePath = useGlob ? asGlob : path;
-        m_tagToPath[pathTag] = usePath;
-        m_pathToTag[usePath.hash()] = pathTag;
-        WriteTag(cacheWriter, pathTag, usePath);
-#if DUMP_CACHE_FILL
-        DumpCacheAdd(pathTag, usePath);
-#endif
-    }
-
-    return true;
-}
-
-void ProjectResourceFactoryBase::BackgroundIndexRecursiveProc(const hecl::ProjectPath& dir,
-                                                              athena::io::YAMLDocWriter& cacheWriter,
-                                                              athena::io::YAMLDocWriter& nameWriter,
-                                                              int level)
-{
-    hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(),
-                                    hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted,
-                                    false, false, true);
-
-    /* Enumerate all items */
-    for (const hecl::DirectoryEnumerator::Entry& ent : dEnum)
-    {
-        hecl::ProjectPath path(dir, ent.m_name);
-        if (ent.m_isDir)
-            BackgroundIndexRecursiveProc(path, cacheWriter, nameWriter, level+1);
-        else
-        {
-            if (!path.isFile())
-                continue;
-
-            /* Read catalog.yaml for .pak directory if exists */
-            if (level == 1 && !ent.m_name.compare(_S("!catalog.yaml")))
-            {
-                ReadCatalog(path, nameWriter);
-                continue;
-            }
-
-            /* Index the regular file */
-            AddFileToIndex(path, cacheWriter);
-        }
-
-        /* bail if cancelled by client */
-        if (!m_backgroundRunning)
-            break;
-    }
-}
-
-void ProjectResourceFactoryBase::BackgroundIndexProc()
-{
-    logvisor::RegisterThreadName("Resource Index Thread");
-
-    hecl::ProjectPath tagCachePath(m_proj->getProjectCookedPath(*m_origSpec), _S("tag_cache.yaml"));
-    hecl::ProjectPath nameCachePath(m_proj->getProjectCookedPath(*m_origSpec), _S("name_cache.yaml"));
-    hecl::ProjectPath specRoot(m_proj->getProjectWorkingPath(), m_origSpec->m_name);
-
-    /* Cache will be overwritten with validated entries afterwards */
-    athena::io::YAMLDocWriter cacheWriter(nullptr);
-    athena::io::YAMLDocWriter nameWriter(nullptr);
-
-    /* Read in tag cache */
-    if (tagCachePath.isFile())
-    {
-        athena::io::FileReader reader(tagCachePath.getAbsolutePath());
-        if (reader.isOpen())
-        {
-            Log.report(logvisor::Info, _S("Cache index of '%s' loading"), m_origSpec->m_name);
-            athena::io::YAMLDocReader cacheReader;
-            if (cacheReader.parse(&reader))
-            {
-                std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
-                size_t tagCount = cacheReader.getRootNode()->m_mapChildren.size();
-                m_tagToPath.reserve(tagCount);
-                m_pathToTag.reserve(tagCount);
-                size_t loadIdx = 0;
-                for (const auto& child : cacheReader.getRootNode()->m_mapChildren)
-                {
-                    const athena::io::YAMLNode& node = *child.second;
-                    unsigned long id = strtoul(child.first.c_str(), nullptr, 16);
-                    hecl::FourCC type(node.m_seqChildren.at(0)->m_scalarString.c_str());
-                    hecl::ProjectPath path(m_proj->getProjectWorkingPath(),
-                        node.m_seqChildren.at(1)->m_scalarString);
-
-                    if (path.isFileOrGlob())
-                    {
-                        SObjectTag pathTag(type, id);
-                        m_tagToPath[pathTag] = path;
-                        m_pathToTag[path.hash()] = pathTag;
-                        WriteTag(cacheWriter, pathTag, path);
-                    }
-                    fprintf(stderr, "\r %" PRISize " / %" PRISize, ++loadIdx, tagCount);
-                }
-                fprintf(stderr, "\n");
-            }
-            Log.report(logvisor::Info, _S("Cache index of '%s' loaded; %d tags"),
-                       m_origSpec->m_name, m_tagToPath.size());
-
-            if (nameCachePath.isFile())
-            {
-                /* Read in name cache */
-                Log.report(logvisor::Info, _S("Name index of '%s' loading"), m_origSpec->m_name);
-                athena::io::FileReader nreader(nameCachePath.getAbsolutePath());
-                athena::io::YAMLDocReader nameReader;
-                if (nameReader.parse(&nreader))
-                {
-                    std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
-                    m_catalogNameToTag.reserve(nameReader.getRootNode()->m_mapChildren.size());
-                    for (const auto& child : nameReader.getRootNode()->m_mapChildren)
-                    {
-                        unsigned long id = strtoul(child.second->m_scalarString.c_str(), nullptr, 16);
-                        auto search = m_tagToPath.find(SObjectTag(FourCC(), uint32_t(id)));
-                        if (search != m_tagToPath.cend())
-                        {
-                            std::string chLower = child.first;
-                            std::transform(chLower.cbegin(), chLower.cend(), chLower.begin(), tolower);
-                            m_catalogNameToTag[chLower] = search->first;
-                            WriteNameTag(nameWriter, search->first, child.first);
-                        }
-                    }
-                }
-                Log.report(logvisor::Info, _S("Name index of '%s' loaded; %d names"),
-                           m_origSpec->m_name, m_catalogNameToTag.size());
-            }
-        }
-    }
-
-    /* Add special original IDs resource if exists (not name-cached to disk) */
-    hecl::ProjectPath oidsPath(specRoot, "!original_ids.yaml");
-    SObjectTag oidsTag = BuildTagFromPath(oidsPath, m_backgroundBlender);
-    if (oidsTag)
-        m_catalogNameToTag["mp1originalids"] = oidsTag;
-
-    Log.report(logvisor::Info, _S("Background index of '%s' started"), m_origSpec->m_name);
-    BackgroundIndexRecursiveProc(specRoot, cacheWriter, nameWriter, 0);
-
-    tagCachePath.makeDirChain(false);
-    athena::io::FileWriter twriter(tagCachePath.getAbsolutePath());
-    cacheWriter.finish(&twriter);
-
-    athena::io::FileWriter nwriter(nameCachePath.getAbsolutePath());
-    nameWriter.finish(&nwriter);
-
-    m_backgroundBlender.shutdown();
-    Log.report(logvisor::Info, _S("Background index of '%s' complete; %d tags, %d names"),
-               m_origSpec->m_name, m_tagToPath.size(), m_catalogNameToTag.size());
-    m_backgroundRunning = false;
-}
-
-void ProjectResourceFactoryBase::CancelBackgroundIndex()
-{
-    m_backgroundRunning = false;
-    if (m_backgroundIndexTh.joinable())
-        m_backgroundIndexTh.join();
-}
-
 void ProjectResourceFactoryBase::BeginBackgroundIndex
     (hecl::Database::Project& proj,
      const hecl::Database::DataSpecEntry& origSpec,
      const hecl::Database::DataSpecEntry& pcSpec)
 {
     CancelBackgroundIndex();
-    Clear();
     m_proj = &proj;
     m_origSpec = &origSpec;
     m_pcSpec = &pcSpec;
     m_cookSpec.reset(pcSpec.m_factory(proj, hecl::Database::DataSpecTool::Cook));
-    m_backgroundRunning = true;
+    return static_cast<DataSpec::SpecBase&>(*m_cookSpec).beginBackgroundIndex();
-    m_backgroundIndexTh =
-        std::thread(std::bind(&ProjectResourceFactoryBase::BackgroundIndexProc, this));
-}
-
-hecl::ProjectPath ProjectResourceFactoryBase::GetCookedPath(const hecl::ProjectPath& working,
-                                                            bool pcTarget) const
-{
-    const hecl::Database::DataSpecEntry* spec = m_origSpec;
-    if (pcTarget)
-        spec = m_cookSpec->overrideDataSpec(working, m_pcSpec, hecl::SharedBlenderToken);
-    if (!spec)
-        return {};
-    return working.getCookedPath(*spec);
 }
 
 bool ProjectResourceFactoryBase::SyncCook(const hecl::ProjectPath& working)
@@ -629,35 +205,6 @@ ProjectResourceFactoryBase::_RemoveTask(const SObjectTag& tag)
     return _RemoveTask(m_asyncLoadMap.find(tag));
 };
 
-bool ProjectResourceFactoryBase::WaitForTagReady(const urde::SObjectTag& tag,
-                                                 const hecl::ProjectPath*& pathOut)
-{
-    std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
-    auto search = m_tagToPath.find(tag);
-    if (search == m_tagToPath.end())
-    {
-        if (m_backgroundRunning)
-        {
-            while (m_backgroundRunning)
-            {
-                lk.unlock();
-                std::this_thread::sleep_for(std::chrono::milliseconds(2));
-                lk.lock();
-                search = m_tagToPath.find(tag);
-                if (search != m_tagToPath.end())
-                    break;
-            }
-            if (search == m_tagToPath.end())
-                return false;
-        }
-        else
-            return false;
-    }
-    lk.unlock();
-    pathOut = &search->second;
-    return true;
-}
-
 bool
 ProjectResourceFactoryBase::PrepForReadSync(const SObjectTag& tag,
                                             const hecl::ProjectPath& path,
@@ -922,108 +469,34 @@ bool ProjectResourceFactoryBase::CanBuild(const urde::SObjectTag& tag)
 
 const urde::SObjectTag* ProjectResourceFactoryBase::GetResourceIdByName(const char* name) const
 {
-    std::string lower = name;
+    return static_cast<DataSpec::SpecBase&>(*m_cookSpec).getResourceIdByName(name);
-    std::transform(lower.cbegin(), lower.cend(), lower.begin(), tolower);
-
-    std::unique_lock<std::mutex> lk(const_cast<ProjectResourceFactoryBase*>(this)->m_backgroundIndexMutex);
-    auto search = m_catalogNameToTag.find(lower);
-    if (search == m_catalogNameToTag.end())
-    {
-        if (m_backgroundRunning)
-        {
-            while (m_backgroundRunning)
-            {
-                lk.unlock();
-                std::this_thread::sleep_for(std::chrono::milliseconds(2));
-                lk.lock();
-                search = m_catalogNameToTag.find(lower);
-                if (search != m_catalogNameToTag.end())
-                    break;
-            }
-            if (search == m_catalogNameToTag.end())
-                return nullptr;
-        }
-        else
-            return nullptr;
-    }
-    return &search->second;
 }
 
 FourCC ProjectResourceFactoryBase::GetResourceTypeById(CAssetId id) const
 {
-    if (!id.IsValid())
+    return static_cast<DataSpec::SpecBase&>(*m_cookSpec).getResourceTypeById(id);
-        return {};
-
-    std::unique_lock<std::mutex> lk(const_cast<ProjectResourceFactoryBase*>(this)->m_backgroundIndexMutex);
-    SObjectTag searchTag = {FourCC(), id};
-    auto search = m_tagToPath.find(searchTag);
-    if (search == m_tagToPath.end())
-    {
-        if (m_backgroundRunning)
-        {
-            while (m_backgroundRunning)
-            {
-                lk.unlock();
-                std::this_thread::sleep_for(std::chrono::milliseconds(2));
-                lk.lock();
-                search = m_tagToPath.find(searchTag);
-                if (search != m_tagToPath.end())
-                    break;
-            }
-            if (search == m_tagToPath.end())
-                return {};
-        }
-        else
-            return {};
-    }
-
-    return search->first.type;
 }
 
 void ProjectResourceFactoryBase::EnumerateResources(const std::function<bool(const SObjectTag&)>& lambda) const
 {
-    std::unique_lock<std::mutex> lk(const_cast<ProjectResourceFactoryBase*>(this)->m_backgroundIndexMutex);
+    return static_cast<DataSpec::SpecBase&>(*m_cookSpec).enumerateResources(lambda);
-    while (m_backgroundRunning)
-    {
-        lk.unlock();
-        std::this_thread::sleep_for(std::chrono::milliseconds(2));
-        lk.lock();
-    }
-    for (const auto& pair : m_tagToPath)
-    {
-        if (!lambda(pair.first))
-            break;
-    }
 }
 
 void ProjectResourceFactoryBase::EnumerateNamedResources(
         const std::function<bool(const std::string&, const SObjectTag&)>& lambda) const
 {
-    std::unique_lock<std::mutex> lk(const_cast<ProjectResourceFactoryBase*>(this)->m_backgroundIndexMutex);
+    return static_cast<DataSpec::SpecBase&>(*m_cookSpec).enumerateNamedResources(lambda);
-    while (m_backgroundRunning)
-    {
-        lk.unlock();
-        std::this_thread::sleep_for(std::chrono::milliseconds(2));
-        lk.lock();
-    }
-    lk.unlock();
-    for (const auto& pair : m_catalogNameToTag)
-    {
-        if (!lambda(pair.first, pair.second))
-            break;
-    }
 }
 
 template <typename ItType>
 bool ProjectResourceFactoryBase::AsyncPumpTask(ItType& it)
 {
     /* Ensure requested resource is in the index */
-    std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
     AsyncTask& task = _GetAsyncTask(it);
-    auto search = m_tagToPath.find(task.x0_tag);
+    hecl::ProjectPath path = static_cast<DataSpec::SpecBase&>(*m_cookSpec).pathFromTag(task.x0_tag);
-    if (search == m_tagToPath.end())
+    if (!path)
     {
-        if (!m_backgroundRunning)
+        if (!static_cast<DataSpec::SpecBase&>(*m_cookSpec).backgroundIndexRunning())
         {
             Log.report(logvisor::Error, _S("unable to find async load resource (%s, %08X)"),
                        task.x0_tag.type.toString().c_str(), task.x0_tag.id);
@@ -1031,8 +504,7 @@ bool ProjectResourceFactoryBase::AsyncPumpTask(ItType& it)
         }
         return true;
     }
-    lk.unlock();
+    task.EnsurePath(task.x0_tag, path);
-    task.EnsurePath(task.x0_tag, search->second);
 
     /* Pump load pipeline (cooking if needed) */
     if (task.AsyncPump())

Editor/ProjectResourceFactoryBase.hpp

@@ -6,6 +6,7 @@
 #include "Runtime/IFactory.hpp"
 #include "Runtime/CFactoryMgr.hpp"
 #include "Runtime/CResFactory.hpp"
+#include "DataSpec/SpecBase.hpp"
 #include "optional.hpp"
 
 #include <thread>
@@ -72,11 +73,6 @@ public:
     };
 
 protected:
-    std::unordered_map<urde::SObjectTag, hecl::ProjectPath> m_tagToPath;
-    std::unordered_map<hecl::Hash, urde::SObjectTag> m_pathToTag;
-    std::unordered_map<std::string, urde::SObjectTag> m_catalogNameToTag;
-    void Clear();
-
     const hecl::Database::Project* m_proj = nullptr;
     const hecl::Database::DataSpecEntry* m_origSpec = nullptr;
     const hecl::Database::DataSpecEntry* m_pcSpec = nullptr;
@@ -84,11 +80,6 @@ protected:
     std::unique_ptr<hecl::Database::IDataSpec> m_cookSpec;
     urde::CFactoryMgr m_factoryMgr;
 
-    hecl::BlenderToken m_backgroundBlender;
-    std::thread m_backgroundIndexTh;
-    std::mutex m_backgroundIndexMutex;
-    bool m_backgroundRunning = false;
-
     std::list<std::shared_ptr<AsyncTask>> m_asyncLoadList;
     std::unordered_map<SObjectTag, std::list<std::shared_ptr<AsyncTask>>::iterator> m_asyncLoadMap;
     std::shared_ptr<AsyncTask>
@@ -111,36 +102,36 @@ protected:
         return **it->second;
     }
 
-    bool WaitForTagReady(const urde::SObjectTag& tag, const hecl::ProjectPath*& pathOut);
     bool
     PrepForReadSync(const SObjectTag& tag,
                     const hecl::ProjectPath& path,
                     std::experimental::optional<athena::io::FileReader>& fr);
 
-    SObjectTag TagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const;    
+    bool WaitForTagReady(const urde::SObjectTag& tag, const hecl::ProjectPath*& pathOut)
+    {
+        return static_cast<DataSpec::SpecBase&>(*m_cookSpec).waitForTagReady(tag, pathOut);
+    }
+    SObjectTag TagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
+    {
+        return static_cast<DataSpec::SpecBase&>(*m_cookSpec).tagFromPath(path, btok);
+    }
     SObjectTag BuildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
     {
-        return static_cast<DataSpec::SpecBase&>(*m_cookSpec).BuildTagFromPath(path, btok);
+        return static_cast<DataSpec::SpecBase&>(*m_cookSpec).buildTagFromPath(path, btok);
+    }
+    void GetTagListForFile(const char* pakName, std::vector<SObjectTag>& out) const
+    {
+        return static_cast<DataSpec::SpecBase&>(*m_cookSpec).getTagListForFile(pakName, out);
+    }
+    void CancelBackgroundIndex()
+    {
+        if (m_cookSpec)
+            return static_cast<DataSpec::SpecBase&>(*m_cookSpec).cancelBackgroundIndex();
     }
-
-    void ReadCatalog(const hecl::ProjectPath& catalogPath,
-                     athena::io::YAMLDocWriter& nameWriter);
-    bool AddFileToIndex(const hecl::ProjectPath& path,
-                        athena::io::YAMLDocWriter& cacheWriter);
-    void BackgroundIndexRecursiveProc(const hecl::ProjectPath& path,
-                                      athena::io::YAMLDocWriter& cacheWriter,
-                                      athena::io::YAMLDocWriter& nameWriter,
-                                      int level);
-    void BackgroundIndexRecursiveCatalogs(const hecl::ProjectPath& path,
-                                          athena::io::YAMLDocWriter& nameWriter,
-                                          int level);
-    void BackgroundIndexProc();
-    void CancelBackgroundIndex();
     void BeginBackgroundIndex(hecl::Database::Project& proj,
-                              const hecl::Database::DataSpecEntry& origSpec,
+         const hecl::Database::DataSpecEntry& origSpec,
-                              const hecl::Database::DataSpecEntry& pcSpec);
+         const hecl::Database::DataSpecEntry& pcSpec);
 
-    hecl::ProjectPath GetCookedPath(const hecl::ProjectPath& working, bool pcTarget) const;
     bool SyncCook(const hecl::ProjectPath& working);
     CFactoryFnReturn BuildSync(const SObjectTag& tag, const hecl::ProjectPath& path,
                                const CVParamTransfer& paramXfer, CObjectReference* selfRef);
@@ -154,6 +145,10 @@ public:
     bool CanBuild(const urde::SObjectTag&);
     const urde::SObjectTag* GetResourceIdByName(const char*) const;
     FourCC GetResourceTypeById(CAssetId id) const;
+    hecl::ProjectPath GetCookedPath(const hecl::ProjectPath& working, bool pcTarget) const
+    {
+        return static_cast<DataSpec::SpecBase&>(*m_cookSpec).getCookedPath(working, pcTarget);
+    }
 
     void EnumerateResources(const std::function<bool(const SObjectTag&)>& lambda) const;
     void EnumerateNamedResources(const std::function<bool(const std::string&, const SObjectTag&)>& lambda) const;

Editor/ProjectResourceFactoryMP1.cpp

@@ -135,19 +135,6 @@ void ProjectResourceFactoryMP1::IndexMP1Resources(hecl::Database::Project& proj,
     m_origIds = sp.GetObj("MP1OriginalIDs");
 }
 
-void ProjectResourceFactoryMP1::GetTagListForFile(const char* pakName, std::vector<SObjectTag>& out) const
-{
-    std::string pathPrefix("MP1/");
-    pathPrefix += pakName;
-    pathPrefix += '/';
-
-    std::unique_lock<std::mutex> lk(
-        const_cast<ProjectResourceFactoryMP1&>(*this).m_backgroundIndexMutex);
-    for (const auto& tag : m_tagToPath)
-        if (!tag.second.getRelativePathUTF8().compare(0, pathPrefix.size(), pathPrefix))
-            out.push_back(tag.first);
-}
-
 void ProjectResourceFactoryMP1::Shutdown()
 {
     m_origIds = TLockedToken<MP1OriginalIDs>();

Editor/ProjectResourceFactoryMP1.hpp

@@ -15,7 +15,6 @@ class ProjectResourceFactoryMP1 : public ProjectResourceFactoryBase
 public:
     ProjectResourceFactoryMP1(hecl::ClientProcess& clientProc);
     void IndexMP1Resources(hecl::Database::Project& proj, CSimplePool& sp);
-    void GetTagListForFile(const char* pakName, std::vector<SObjectTag>& out) const;
     void Shutdown();
 
     CAssetId TranslateOriginalToNew(CAssetId id) const;

Runtime/CDvdFile.hpp

@@ -2,6 +2,7 @@
 #define __URDE_CDVDFILE_HPP__
 
 #include "RetroTypes.hpp"
+#include "athena/FileReader.hpp"
 
 #include <thread>
 #include <mutex>

Runtime/CSortedLists.hpp

@@ -20,7 +20,7 @@ enum ESortedList
 struct SSortedList
 {
     s16 x0_ids[1024];
-    u32 x800_size;
+    u32 x800_size = 0;
     void Reset() {std::fill(std::begin(x0_ids), std::end(x0_ids), -1);}
     SSortedList() {Reset();}
 };

Runtime/CStaticInterference.cpp

@@ -1,4 +1,5 @@
 #include "CStaticInterference.hpp"
+#include "zeus/Math.hpp"
 
 namespace urde
 {
@@ -24,7 +25,7 @@ void CStaticInterference::Update(CStateManager&, float dt)
     newSources.reserve(m_sources.size());
     for (CStaticInterferenceSource& src : m_sources)
     {
-        if (src.timeLeft >= 0.0)
+        if (src.timeLeft >= 0.f)
         {
             src.timeLeft -= dt;
             newSources.push_back(src);
@@ -35,8 +36,8 @@ void CStaticInterference::Update(CStateManager&, float dt)
 
 float CStaticInterference::GetTotalInterference() const
 {
-    float validAccum = 0.0;
+    float validAccum = 0.f;
-    float invalidAccum = 0.0;
+    float invalidAccum = 0.f;
     for (const CStaticInterferenceSource& src : m_sources)
     {
         if (src.id == kInvalidUniqueId)
@@ -44,11 +45,11 @@ float CStaticInterference::GetTotalInterference() const
         else
             validAccum += src.magnitude;
     }
-    if (validAccum > 0.80000001)
+    if (validAccum > 0.80000001f)
-        validAccum = 0.80000001;
+        validAccum = 0.80000001f;
     validAccum += invalidAccum;
-    if (validAccum > 1.0)
+    if (validAccum > 1.f)
-        return 1.0;
+        return 1.f;
     return validAccum;
 }
 

Runtime/Camera/CCameraSpline.hpp

@@ -2,6 +2,7 @@
 #define __URDE_CCAMERASPLINE_HPP__
 
 #include "World/CEntityInfo.hpp"
+#include "zeus/CVector3f.hpp"
 
 namespace urde
 {

Runtime/Character/CBodyStateCmdMgr.cpp

@@ -1,4 +1,5 @@
 #include "CBodyStateCmdMgr.hpp"
+#include <cfloat>
 
 namespace urde
 {

Runtime/GuiSys/CHudBossEnergyInterface.hpp

@@ -2,6 +2,7 @@
 #define __URDE_CHUDBOSSENERGYINTERFACE_HPP__
 
 #include "RetroTypes.hpp"
+#include "zeus/CVector3f.hpp"
 
 namespace urde
 {

Runtime/GuiSys/CHudEnergyInterface.hpp

@@ -3,6 +3,7 @@
 
 #include "RetroTypes.hpp"
 #include "CHudInterface.hpp"
+#include "zeus/CVector3f.hpp"
 
 namespace urde
 {

Runtime/GuiSys/CHudVisorBeamMenu.hpp

@@ -2,6 +2,7 @@
 #define __URDE_CHUDVISORBEAMMENU_HPP__
 
 #include "RetroTypes.hpp"
+#include <cfloat>
 
 namespace urde
 {

Runtime/MP1/CNESEmulator.hpp

@@ -2,6 +2,7 @@
 #define __URDE_CNESEMULATOR_HPP__
 
 #include "RetroTypes.hpp"
+#include "zeus/CColor.hpp"
 
 namespace urde
 {

Runtime/RetroTypes.hpp

@@ -4,10 +4,11 @@
 #include <vector>
 #include <utility>
 #include <string>
+#include <functional>
 #include "GCNTypes.hpp"
 #include "rstl.hpp"
-#include "DataSpec/DNACommon/DNACommon.hpp"
 #include "IOStreams.hpp"
+#include "hecl/hecl.hpp"
 
 namespace urde
 {
@@ -24,10 +25,9 @@ public:
     explicit CAssetId(CInputStream& in);
     bool IsValid() const { return id != UINT64_MAX; }
     u64 Value() const { return id; }
-    void Assign(u64 v) {  id = (v == UINT32_MAX ? UINT64_MAX : (v == 0 ? UINT64_MAX : v)); }
+    void Assign(u64 v) { id = (v == UINT32_MAX ? UINT64_MAX : (v == 0 ? UINT64_MAX : v)); }
     void Reset() { id = UINT64_MAX; }
     void PutTo(COutputStream& out);
-
     bool operator==(const CAssetId& other) const { return id == other.id; }
     bool operator!=(const CAssetId& other) const { return id != other.id; }
     bool operator<(const CAssetId& other) const { return id < other.id; }
@@ -39,6 +39,7 @@ struct SObjectTag
 {
     FourCC type;
     CAssetId id;
+
     operator bool() const { return id.IsValid(); }
     bool operator!=(const SObjectTag& other) const { return id != other.id; }
     bool operator==(const SObjectTag& other) const { return id == other.id; }
@@ -65,11 +66,11 @@ struct TEditorId
     u8 LayerNum() const { return u8((id >> 26) & 0x3f); }
     u16 AreaNum() const { return u16((id >> 16) & 0x3ff); }
     u16 Id() const { return u16(id & 0xffff); }
-
     bool operator<(const TEditorId& other) const { return (id & 0x3ffffff) < (other.id & 0x3ffffff); }
     bool operator!=(const TEditorId& other) const { return (id & 0x3ffffff) != (other.id & 0x3ffffff); }
     bool operator==(const TEditorId& other) const { return (id & 0x3ffffff) == (other.id & 0x3ffffff); }
 };
+
 #define kInvalidEditorId TEditorId()
 
 struct TUniqueId
@@ -77,9 +78,8 @@ struct TUniqueId
     TUniqueId() = default;
     TUniqueId(u16 value, u16 version) : id(value | (version << 10)) {}
     u16 id = u16(-1);
-
+    u16 Version() const { return u16((id >> 10) & 0x3f); }
-    u16 Version() const { return u16((id >> 10)  & 0x3f);}
+    u16 Value() const { return u16(id & 0x3ff); }
-    u16 Value() const { return u16(id & 0x3ff);}
     bool operator<(const TUniqueId& other) const { return (id < other.id); }
     bool operator!=(const TUniqueId& other) const { return (id != other.id); }
     bool operator==(const TUniqueId& other) const { return (id == other.id); }
@@ -90,7 +90,6 @@ struct TUniqueId
 using TAreaId = s32;
 
 #define kInvalidAreaId TAreaId(-1)
-}
 
 #if 0
 template <class T, size_t N>
@@ -113,7 +112,7 @@ public:
 };
 #endif
 
-template <class T>
+template<class T>
 T GetAverage(const T* v, s32 count)
 {
     T r = v[0];
@@ -123,11 +122,12 @@ T GetAverage(const T* v, s32 count)
     return r / count;
 }
 
-template <class T, size_t N>
+template<class T, size_t N>
 class TReservedAverage : rstl::reserved_vector<T, N>
 {
 public:
     TReservedAverage() = default;
+
     TReservedAverage(const T& t) { rstl::reserved_vector<T, N>::resize(N, t); }
 
     void AddValue(const T& t)
@@ -135,8 +135,7 @@ public:
         if (this->size() < N)
         {
             this->insert(this->begin(), t);
-        }
+        } else
-        else
         {
             this->pop_back();
             this->insert(this->begin(), t);
@@ -148,7 +147,7 @@ public:
         if (this->empty())
             return {};
 
-        return {::GetAverage<T>(this->data(), this->size())};
+        return {urde::GetAverage<T>(this->data(), this->size())};
     }
 
     rstl::optional_object<T> GetEntry(int i) const
@@ -163,18 +162,20 @@ public:
     size_t Size() const { return this->size(); }
 };
 
+}
+
 namespace std
 {
 template <>
 struct hash<urde::SObjectTag>
 {
-    inline size_t operator()(const urde::SObjectTag& tag) const { return tag.id.Value(); }
+    size_t operator()(const urde::SObjectTag& tag) const noexcept { return tag.id.Value(); }
 };
 
 template <>
 struct hash<urde::CAssetId>
 {
-    inline size_t operator()(const urde::CAssetId& id) const { return id.Value(); }
+    size_t operator()(const urde::CAssetId& id) const noexcept { return id.Value(); }
 };
 }
 

Runtime/World/CAiFuncMap.hpp

@@ -2,6 +2,7 @@
 #define __URDE_CAIFUNCMAP_HPP__
 
 #include "RetroTypes.hpp"
+#include <unordered_map>
 
 namespace urde
 {

Runtime/World/CVisorFlare.hpp

@@ -3,6 +3,7 @@
 
 #include "RetroTypes.hpp"
 #include "CToken.hpp"
+#include "zeus/CColor.hpp"
 
 namespace urde
 {

Runtime/World/CWorldLight.cpp

@@ -1,4 +1,5 @@
 #include "CWorldLight.hpp"
+#include <cfloat>
 
 namespace urde
 {

hecl

@@ -1 +1 @@
-Subproject commit 23a08dcf40456c59ff40710d12241a8a8735fe5e
+Subproject commit 70b73855bf906c81d18bdf70c9b701bba996314e