From e274cd12b9f7e90fd2fcdb7ea9ef49e84ef3e43b Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Tue, 24 Oct 2017 21:47:49 -1000 Subject: [PATCH] Refactorings to support .upak generation --- DataSpec/DNAMP1/ANCS.cpp | 40 +- DataSpec/DNAMP1/DNAMP1.cpp | 2 +- DataSpec/DNAMP1/MLVL.cpp | 13 +- DataSpec/DNAMP1/MREA.cpp | 4 +- DataSpec/DNAMP2/DNAMP2.cpp | 2 +- DataSpec/SpecBase.cpp | 770 ++++++++++++++++++++- DataSpec/SpecBase.hpp | 88 ++- DataSpec/SpecMP1.cpp | 220 +++++- DataSpec/SpecMP2.cpp | 2 +- DataSpec/SpecMP3.cpp | 2 +- Editor/ProjectResourceFactoryBase.cpp | 546 +-------------- Editor/ProjectResourceFactoryBase.hpp | 55 +- Editor/ProjectResourceFactoryMP1.cpp | 13 - Editor/ProjectResourceFactoryMP1.hpp | 1 - Runtime/CDvdFile.hpp | 1 + Runtime/CSortedLists.hpp | 2 +- Runtime/CStaticInterference.cpp | 15 +- Runtime/Camera/CCameraSpline.hpp | 1 + Runtime/Character/CBodyStateCmdMgr.cpp | 1 + Runtime/GuiSys/CHudBossEnergyInterface.hpp | 1 + Runtime/GuiSys/CHudEnergyInterface.hpp | 1 + Runtime/GuiSys/CHudVisorBeamMenu.hpp | 1 + Runtime/MP1/CNESEmulator.hpp | 1 + Runtime/RetroTypes.hpp | 31 +- Runtime/World/CAiFuncMap.hpp | 1 + Runtime/World/CVisorFlare.hpp | 1 + Runtime/World/CWorldLight.cpp | 1 + hecl | 2 +- 28 files changed, 1174 insertions(+), 644 deletions(-) diff --git a/DataSpec/DNAMP1/ANCS.cpp b/DataSpec/DNAMP1/ANCS.cpp index c202d28d9..67655d8b5 100644 --- a/DataSpec/DNAMP1/ANCS.cpp +++ b/DataSpec/DNAMP1/ANCS.cpp @@ -1130,7 +1130,8 @@ bool ANCS::Cook(const hecl::ProjectPath& outPath, if (sub.overlayMeshes.size()) { ch.cmdlOverlay = sub.overlayMeshes[0].second; - ch.cskrOverlay = inPath.ensureAuxInfo(chSysName.sys_str() + _S(".over.CSKR")); + ch.cskrOverlay = inPath.ensureAuxInfo(chSysName.sys_str() + _S('.') + + sub.overlayMeshes[0].first + _S(".CSKR")); } break; @@ -1206,7 +1207,15 @@ bool ANCS::CookCSKR(const hecl::ProjectPath& outPath, { hecl::SystemString subName(inPath.getAuxInfo().begin(), inPath.getAuxInfo().end() - 5); + hecl::SystemString overName; + auto dotPos = subName.rfind(_S('.')); + if (dotPos != hecl::SystemString::npos) + { + overName = hecl::SystemString(subName.begin() + dotPos + 1, subName.end()); + subName = hecl::SystemString(subName.begin(), subName.begin() + dotPos); + } hecl::SystemUTF8View subNameView(subName); + hecl::SystemUTF8View overNameView(overName); /* Build bone ID map */ std::unordered_map boneIdMap; @@ -1227,15 +1236,30 @@ bool ANCS::CookCSKR(const hecl::ProjectPath& outPath, if (!subtype) Log.report(logvisor::Fatal, _S("unable to find subtype '%s'"), subName.c_str()); - const hecl::ProjectPath& modelPath = subtype->mesh; + const hecl::ProjectPath* modelPath = nullptr; + if (overName.empty()) + { + modelPath = &subtype->mesh; + } + else + { + for (const auto& overlay : subtype->overlayMeshes) + if (!overlay.first.compare(overNameView.str())) + { + modelPath = &overlay.second; + break; + } + } + if (!modelPath) + Log.report(logvisor::Fatal, _S("unable to resolve model path of %s:%s"), subName.c_str(), overName.c_str()); - if (!modelPath.isFile()) - Log.report(logvisor::Fatal, _S("unable to resolve '%s'"), modelPath.getRelativePath().c_str()); + if (!modelPath->isFile()) + Log.report(logvisor::Fatal, _S("unable to resolve '%s'"), modelPath->getRelativePath().c_str()); - hecl::ProjectPath skinIntPath = modelPath.getCookedPath(SpecEntMP1PC).getWithExtension(_S(".skinint")); - if (!skinIntPath.isFileOrGlob() || skinIntPath.getModtime() < modelPath.getModtime()) - if (!modelCookFunc(modelPath)) - Log.report(logvisor::Fatal, _S("unable to cook '%s'"), modelPath.getRelativePath().c_str()); + hecl::ProjectPath skinIntPath = modelPath->getCookedPath(SpecEntMP1PC).getWithExtension(_S(".skinint")); + if (!skinIntPath.isFileOrGlob() || skinIntPath.getModtime() < modelPath->getModtime()) + if (!modelCookFunc(*modelPath)) + Log.report(logvisor::Fatal, _S("unable to cook '%s'"), modelPath->getRelativePath().c_str()); athena::io::FileReader skinIO(skinIntPath.getAbsolutePath(), 1024*32, false); if (skinIO.hasError()) diff --git a/DataSpec/DNAMP1/DNAMP1.cpp b/DataSpec/DNAMP1/DNAMP1.cpp index 5793d616b..91500689d 100644 --- a/DataSpec/DNAMP1/DNAMP1.cpp +++ b/DataSpec/DNAMP1/DNAMP1.cpp @@ -244,7 +244,7 @@ void PAKBridge::addCMDLRigPairs(PAKRouter& pakRouter, if (ci.cmdlOverlay && ci.cskrOverlay) { addTo[ci.cmdlOverlay] = std::make_pair(ci.cskrOverlay, ci.cinf); - cskrCinfToAncs[ci.cskrOverlay] = std::make_pair(entry.second.id, hecl::Format("%s.over.CSKR", ci.name.c_str())); + cskrCinfToAncs[ci.cskrOverlay] = std::make_pair(entry.second.id, hecl::Format("%s.OVER.CSKR", ci.name.c_str())); PAK::Entry* cmdlEnt = (PAK::Entry*)m_pak.lookupEntry(ci.cmdlOverlay); PAK::Entry* cskrEnt = (PAK::Entry*)m_pak.lookupEntry(ci.cskrOverlay); cmdlEnt->name = hecl::Format("ANCS_%08X_%s_overmodel", entry.first.toUint32(), ci.name.c_str()); diff --git a/DataSpec/DNAMP1/MLVL.cpp b/DataSpec/DNAMP1/MLVL.cpp index ef45afdbe..0fb062fe9 100644 --- a/DataSpec/DNAMP1/MLVL.cpp +++ b/DataSpec/DNAMP1/MLVL.cpp @@ -73,9 +73,10 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat hecl::ProjectPath namePath(inPath.getParentPath(), _S("!name.yaml")); if (namePath.isFile()) mlvl.worldNameId = namePath; - hecl::ProjectPath savwPath = inPath.ensureAuxInfo(_S("SAVW")); + hecl::ProjectPath globPath = inPath.getWithExtension(_S(".*"), true); + hecl::ProjectPath savwPath = globPath.ensureAuxInfo(_S("SAVW")); mlvl.saveWorldId = savwPath; - hecl::ProjectPath mapwPath = inPath.ensureAuxInfo(_S("MAPW")); + hecl::ProjectPath mapwPath = globPath.ensureAuxInfo(_S("MAPW")); mlvl.worldMap = mapwPath; size_t areaIdx = 0; @@ -269,7 +270,7 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat if (addedPaths.find(path.hash()) == addedPaths.cend()) { addedPaths.insert(path.hash()); - urde::SObjectTag tag = g_curSpec->BuildTagFromPath(path, btok); + urde::SObjectTag tag = g_curSpec->buildTagFromPath(path, btok); if (tag.id.IsValid()) areaOut.deps.emplace_back(tag.id.Value(), tag.type); } @@ -308,13 +309,13 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat if (addedPaths.find(path.hash()) == addedPaths.cend()) { addedPaths.insert(path.hash()); - urde::SObjectTag tag = g_curSpec->BuildTagFromPath(path, btok); + urde::SObjectTag tag = g_curSpec->buildTagFromPath(path, btok); if (tag.id.IsValid()) areaOut.deps.emplace_back(tag.id.Value(), tag.type); } } - urde::SObjectTag tag = g_curSpec->BuildTagFromPath(areaPath, btok); + urde::SObjectTag tag = g_curSpec->buildTagFromPath(areaPath, btok); if (tag.id.IsValid()) areaOut.deps.emplace_back(tag.id.Value(), tag.type); } @@ -359,7 +360,7 @@ bool MLVL::CookMAPW(const hecl::ProjectPath& outPath, /* Area map */ hecl::ProjectPath mapPath(area.path, _S("/!map.blend")); if (mapPath.isFile()) - mapaTags.push_back(g_curSpec->BuildTagFromPath(mapPath, btok)); + mapaTags.push_back(g_curSpec->buildTagFromPath(mapPath, btok)); } /* Write out MAPW */ diff --git a/DataSpec/DNAMP1/MREA.cpp b/DataSpec/DNAMP1/MREA.cpp index 675c2a41f..397eb7cd3 100644 --- a/DataSpec/DNAMP1/MREA.cpp +++ b/DataSpec/DNAMP1/MREA.cpp @@ -562,9 +562,9 @@ bool MREA::PCCook(const hecl::ProjectPath& outPath, hecl::SNPrintf(thrIdx, 16, _S("%d"), hecl::ClientProcess::GetThreadWorkerIdx()); hecl::SystemChar parPid[32]; #if _WIN32 - hecl::SNPrintf(parPid, 32, _S("%ullX"), reinterpret_cast(GetCurrentProcess())); + hecl::SNPrintf(parPid, 32, _S("%lluX"), reinterpret_cast(GetCurrentProcess())); #else - hecl::SNPrintf(parPid, 32, _S("%ullX"), reinterpret_cast(getpid())); + hecl::SNPrintf(parPid, 32, _S("%lluX"), (unsigned long long)getpid()); #endif const hecl::SystemChar* args[] = {VisiGenPath.c_str(), visiIntOut.getAbsolutePath().c_str(), diff --git a/DataSpec/DNAMP2/DNAMP2.cpp b/DataSpec/DNAMP2/DNAMP2.cpp index 066096c28..054e56be7 100644 --- a/DataSpec/DNAMP2/DNAMP2.cpp +++ b/DataSpec/DNAMP2/DNAMP2.cpp @@ -191,7 +191,7 @@ void PAKBridge::addCMDLRigPairs(PAKRouter& pakRouter, if (ci.cmdlOverlay) { addTo[ci.cmdlOverlay] = std::make_pair(ci.cskrOverlay, ci.cinf); - cskrCinfToAncs[ci.cskrOverlay] = std::make_pair(entry.second.id, hecl::Format("%s.over.CSKR", ci.name.c_str())); + cskrCinfToAncs[ci.cskrOverlay] = std::make_pair(entry.second.id, hecl::Format("%s.OVER.CSKR", ci.name.c_str())); } } } diff --git a/DataSpec/SpecBase.cpp b/DataSpec/SpecBase.cpp index c52d9e873..9d3b20770 100644 --- a/DataSpec/SpecBase.cpp +++ b/DataSpec/SpecBase.cpp @@ -5,14 +5,15 @@ #include "SpecBase.hpp" #include "Blender/BlenderSupport.hpp" -#include "hecl/Blender/BlenderConnection.hpp" #include "DNACommon/DNACommon.hpp" #include "DNACommon/TXTR.hpp" #include "AssetNameMap.hpp" +#include "hecl/ClientProcess.hpp" -#include #include +#define DUMP_CACHE_FILL 1 + namespace DataSpec { @@ -43,6 +44,11 @@ SpecBase::SpecBase(const hecl::Database::DataSpecEntry* specEntry, hecl::Databas DataSpec::UniqueIDBridge::setThreadProject(m_project); } +SpecBase::~SpecBase() +{ + cancelBackgroundIndex(); +} + static const hecl::SystemString regNONE = _S(""); static const hecl::SystemString regE = _S("NTSC"); static const hecl::SystemString regJ = _S("NTSC-J"); @@ -133,7 +139,15 @@ static bool IsPathAudioGroup(const hecl::ProjectPath& path) !path.getWithExtension(_S(".proj"), true).isFile() || !path.getWithExtension(_S(".sdir"), true).isFile() || !path.getWithExtension(_S(".samp"), true).isFile()) + { + if (path.isFile() && + !hecl::StrCmp(_S("proj"), path.getLastComponentExt()) && + path.getWithExtension(_S(".pool"), true).isFile() && + path.getWithExtension(_S(".sdir"), true).isFile() && + path.getWithExtension(_S(".samp"), true).isFile()) + return true; return false; + } return true; } @@ -142,7 +156,13 @@ static bool IsPathSong(const hecl::ProjectPath& path) if (path.getPathType() != hecl::ProjectPath::Type::Glob || !path.getWithExtension(_S(".mid"), true).isFile() || !path.getWithExtension(_S(".yaml"), true).isFile()) + { + if (path.isFile() && + !hecl::StrCmp(_S("mid"), path.getLastComponentExt()) && + path.getWithExtension(_S(".yaml"), true).isFile()) + return true; return false; + } return true; } @@ -359,10 +379,11 @@ void SpecBase::flattenDependencies(const hecl::ProjectPath& path, const auto& arm = actor.armatures[sub.armature]; hecl::SystemStringView armSysName(arm.name); pathsOut.push_back(path.ensureAuxInfo(armSysName.sys_str() + _S(".CINF"))); - if (sub.overlayMeshes.size()) + for (const auto& overlay : sub.overlayMeshes) { - pathsOut.push_back(sub.overlayMeshes[0].second); - pathsOut.push_back(path.ensureAuxInfo(chSysName.sys_str() + _S(".over.CSKR"))); + pathsOut.push_back(overlay.second); + pathsOut.push_back(path.ensureAuxInfo(chSysName.sys_str() + _S('.') + + overlay.first + _S(".CSKR"))); } } } @@ -402,13 +423,180 @@ void SpecBase::flattenDependencies(const UniqueID64& id, std::vector& listOut, + const hecl::ProjectPath& path, + hecl::BlenderToken& btok) { + hecl::DirectoryEnumerator dEnum(path.getAbsolutePath(), + hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted, false, false, true); + for (const auto& ent : dEnum) + { + hecl::ProjectPath childPath(path, ent.m_name); + if (ent.m_isDir) + recursiveBuildResourceList(listOut, childPath, btok); + else if (urde::SObjectTag tag = tagFromPath(childPath, btok)) + listOut.push_back(tag); + } +} + +void SpecBase::copyBuildListData(std::vector>& fileIndex, + const std::vector& buildList, + const hecl::Database::DataSpecEntry* entry, + bool fast, FProgress progress, athena::io::FileWriter& pakOut) +{ + fileIndex.reserve(buildList.size()); + size_t loadIdx = 0; + for (const auto& tag : buildList) + { + fprintf(stderr, "\r %" PRISize " / %" PRISize " %.4s %08X", ++loadIdx, buildList.size(), + tag.type.getChars(), (unsigned int)tag.id.Value()); + + fileIndex.emplace_back(); + auto& thisIdx = fileIndex.back(); + hecl::ProjectPath path = pathFromTag(tag); + hecl::ProjectPath cooked = getCookedPath(path, true); + athena::io::FileReader r(cooked.getAbsolutePath()); + if (r.hasError()) + Log.report(logvisor::Fatal, _S("Unable to open resource %s"), cooked.getRelativePath().c_str()); + atUint64 size = r.length(); + auto data = r.readUBytes(size); + auto compData = compressPakData(tag, data.get(), size); + if (compData.first) + { + std::get<0>(thisIdx) = pakOut.position(); + std::get<1>(thisIdx) = ROUND_UP_32(compData.second + 4); + std::get<2>(thisIdx) = true; + pakOut.writeUint32Big(atUint32(size)); + pakOut.writeUBytes(compData.first.get(), compData.second); + for (atUint64 i = compData.second + 4 ; i < std::get<1>(thisIdx) ; ++i) + pakOut.writeUByte(0xff); + } + else + { + std::get<0>(thisIdx) = pakOut.position(); + std::get<1>(thisIdx) = ROUND_UP_32(size); + std::get<2>(thisIdx) = false; + pakOut.writeUBytes(data.get(), size); + for (atUint64 i = size ; i < std::get<1>(thisIdx) ; ++i) + pakOut.writeUByte(0xff); + } + } + fprintf(stderr, "\n"); +} + +void SpecBase::doPackage(const hecl::ProjectPath& path, const hecl::Database::DataSpecEntry* entry, + bool fast, hecl::BlenderToken& btok, FProgress progress, hecl::ClientProcess* cp) +{ + /* Prepare complete resource index */ + if (!m_backgroundRunning && m_tagToPath.empty()) + beginBackgroundIndex(); + waitForIndexComplete(); + + /* Name pak based on root-relative components */ + auto components = path.getPathComponents(); + if (components.size() <= 1) + return; + hecl::ProjectPath outPath(m_project.getProjectWorkingPath(), + _S("out/") + components[0] + _S("/") + components[1] + _S(".upak")); + outPath.makeDirChain(false); + hecl::SystemString tmpPath = outPath.getAbsolutePath() + _S("~"); + + /* Output file */ + athena::io::FileWriter pakOut(tmpPath); + std::vector buildList; + atUint64 resTableOffset = 0; + + if (path.getPathType() == hecl::ProjectPath::Type::File && + !hecl::StrCmp(path.getLastComponent(), _S("!world.blend"))) /* World PAK */ + { + /* Force-cook MLVL and write resource list structure */ + m_project.cookPath(path, progress, false, true, fast); + hecl::ProjectPath cooked = getCookedPath(path, true); + buildWorldPakList(path, cooked, btok, pakOut, buildList, resTableOffset); + if (int64_t rem = pakOut.position() % 32) + for (int64_t i=0 ; i<32-rem ; ++i) + pakOut.writeUByte(0xff); + } + else if (path.getPathType() == hecl::ProjectPath::Type::Directory) /* General PAK */ + { + /* Build resource list */ + recursiveBuildResourceList(buildList, path, btok); + std::vector> nameList; + + /* Build name list */ + for (const auto& item : buildList) + { + auto search = m_catalogTagToName.find(item); + if (search != m_catalogTagToName.end()) + nameList.emplace_back(item, search->second); + } + + /* Write resource list structure */ + buildPakList(btok, pakOut, buildList, nameList, resTableOffset); + if (int64_t rem = pakOut.position() % 32) + for (int64_t i=0 ; i<32-rem ; ++i) + pakOut.writeUByte(0xff); + } + + /* Async cook resource list if using ClientProcess */ + if (cp) + { + Log.report(logvisor::Info, _S("Validating resources")); + size_t loadIdx = 0; + for (auto& tag : buildList) + { + fprintf(stderr, "\r %" PRISize " / %" PRISize " %.4s %08X", ++loadIdx, buildList.size(), + tag.type.getChars(), (unsigned int)tag.id.Value()); + hecl::ProjectPath depPath = pathFromTag(tag); + if (!depPath) + { + fprintf(stderr, "\n"); + Log.report(logvisor::Fatal, _S("Unable to resolve %.4s %08X"), + tag.type.getChars(), tag.id.Value()); + } + m_project.cookPath(depPath, progress, false, false, fast, cp); + } + fprintf(stderr, "\n"); + Log.report(logvisor::Info, _S("Waiting for remaining cook transactions")); + cp->waitUntilComplete(); + } + + /* Write resource data and build file index */ + std::vector> fileIndex; + Log.report(logvisor::Info, _S("Copying data into %s"), outPath.getRelativePath().c_str()); + copyBuildListData(fileIndex, buildList, entry, fast, progress, pakOut); + + /* Write file index */ + writePakFileIndex(pakOut, buildList, fileIndex, resTableOffset); + pakOut.close(); + + /* Atomic rename */ + hecl::Rename(tmpPath.c_str(), outPath.getAbsolutePath().c_str()); +} + +hecl::ProjectPath SpecBase::getCookedPath(const hecl::ProjectPath& working, bool pcTarget) const +{ + const hecl::Database::DataSpecEntry* spec = &getOriginalSpec(); + if (pcTarget) + spec = overrideDataSpec(working, getDataSpecEntry(), hecl::SharedBlenderToken); + if (!spec) + return {}; + return working.getCookedPath(*spec); } static void PNGErr(png_structp png, png_const_charp msg) @@ -427,7 +615,7 @@ static inline uint8_t Convert4To8(uint8_t v) return (v << 4) | v; } -void SpecBase::ExtractRandomStaticEntropy(const uint8_t* buf, const hecl::ProjectPath& noAramPath) +void SpecBase::extractRandomStaticEntropy(const uint8_t* buf, const hecl::ProjectPath& noAramPath) { hecl::ProjectPath entropyPath(noAramPath, _S("RandomStaticEntropy.png")); hecl::ProjectPath catalogPath(noAramPath, _S("!catalog.yaml")); @@ -477,4 +665,570 @@ void SpecBase::ExtractRandomStaticEntropy(const uint8_t* buf, const hecl::Projec fclose(fp); } +void SpecBase::clearTagCache() +{ + m_tagToPath.clear(); + m_pathToTag.clear(); + m_catalogNameToTag.clear(); + m_catalogTagToName.clear(); +} + +hecl::ProjectPath SpecBase::pathFromTag(const urde::SObjectTag& tag) const +{ + std::unique_lock lk(const_cast(*this).m_backgroundIndexMutex); + auto search = m_tagToPath.find(tag); + if (search != m_tagToPath.cend()) + return search->second; + return {}; +} + +urde::SObjectTag SpecBase::tagFromPath(const hecl::ProjectPath& path, + hecl::BlenderToken& btok) const +{ + auto search = m_pathToTag.find(path.hash()); + if (search != m_pathToTag.cend()) + return search->second; + return buildTagFromPath(path, btok); +} + +bool SpecBase::waitForTagReady(const urde::SObjectTag& tag, const hecl::ProjectPath*& pathOut) +{ + std::unique_lock lk(m_backgroundIndexMutex); + auto search = m_tagToPath.find(tag); + if (search == m_tagToPath.end()) + { + if (m_backgroundRunning) + { + while (m_backgroundRunning) + { + lk.unlock(); + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + lk.lock(); + search = m_tagToPath.find(tag); + if (search != m_tagToPath.end()) + break; + } + if (search == m_tagToPath.end()) + return false; + } + else + return false; + } + lk.unlock(); + pathOut = &search->second; + return true; +} + +const urde::SObjectTag* SpecBase::getResourceIdByName(const char* name) const +{ + std::string lower = name; + std::transform(lower.cbegin(), lower.cend(), lower.begin(), tolower); + + std::unique_lock lk(const_cast(*this).m_backgroundIndexMutex); + auto search = m_catalogNameToTag.find(lower); + if (search == m_catalogNameToTag.end()) + { + if (m_backgroundRunning) + { + while (m_backgroundRunning) + { + lk.unlock(); + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + lk.lock(); + search = m_catalogNameToTag.find(lower); + if (search != m_catalogNameToTag.end()) + break; + } + if (search == m_catalogNameToTag.end()) + return nullptr; + } + else + return nullptr; + } + return &search->second; +} + +FourCC SpecBase::getResourceTypeById(urde::CAssetId id) const +{ + if (!id.IsValid()) + return {}; + + std::unique_lock lk(const_cast(*this).m_backgroundIndexMutex); + urde::SObjectTag searchTag = {FourCC(), id}; + auto search = m_tagToPath.find(searchTag); + if (search == m_tagToPath.end()) + { + if (m_backgroundRunning) + { + while (m_backgroundRunning) + { + lk.unlock(); + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + lk.lock(); + search = m_tagToPath.find(searchTag); + if (search != m_tagToPath.end()) + break; + } + if (search == m_tagToPath.end()) + return {}; + } + else + return {}; + } + + return search->first.type; +} + +void SpecBase::enumerateResources(const std::function& lambda) const +{ + waitForIndexComplete(); + for (const auto& pair : m_tagToPath) + { + if (!lambda(pair.first)) + break; + } +} + +void SpecBase::enumerateNamedResources( + const std::function& lambda) const +{ + waitForIndexComplete(); + for (const auto& pair : m_catalogNameToTag) + { + if (!lambda(pair.first, pair.second)) + break; + } +} + +static void WriteTag(athena::io::YAMLDocWriter& cacheWriter, + const urde::SObjectTag& pathTag, const hecl::ProjectPath& path) +{ + char idStr[9]; + snprintf(idStr, 9, "%08X", uint32_t(pathTag.id.Value())); + if (auto v = cacheWriter.enterSubVector(idStr)) + { + cacheWriter.writeString(nullptr, pathTag.type.toString().c_str()); + cacheWriter.writeString(nullptr, path.getAuxInfo().size() ? + (path.getRelativePathUTF8() + '|' + path.getAuxInfoUTF8()) : + path.getRelativePathUTF8()); + } +} + +static void WriteNameTag(athena::io::YAMLDocWriter& nameWriter, + const urde::SObjectTag& pathTag, + const std::string& name) +{ + char idStr[9]; + snprintf(idStr, 9, "%08X", uint32_t(pathTag.id.Value())); + nameWriter.writeString(name.c_str(), idStr); +} + +void SpecBase::readCatalog(const hecl::ProjectPath& catalogPath, + athena::io::YAMLDocWriter& nameWriter) +{ + athena::io::FileReader freader(catalogPath.getAbsolutePath()); + if (!freader.isOpen()) + return; + + athena::io::YAMLDocReader reader; + bool res = reader.parse(&freader); + if (!res) + return; + + const athena::io::YAMLNode* root = reader.getRootNode(); + for (const auto& p : root->m_mapChildren) + { + /* Hash as lowercase since lookup is case-insensitive */ + std::string pLower = p.first; + std::transform(pLower.cbegin(), pLower.cend(), pLower.begin(), tolower); + + /* Avoid redundant filesystem access for re-caches */ + if (m_catalogNameToTag.find(pLower) != m_catalogNameToTag.cend()) + continue; + + athena::io::YAMLNode& node = *p.second; + hecl::ProjectPath path; + if (node.m_type == YAML_SCALAR_NODE) + { + path = hecl::ProjectPath(m_project.getProjectWorkingPath(), node.m_scalarString); + } + else if (node.m_type == YAML_SEQUENCE_NODE) + { + if (node.m_seqChildren.size() >= 2) + path = hecl::ProjectPath(m_project.getProjectWorkingPath(), node.m_seqChildren[0]->m_scalarString). + ensureAuxInfo(node.m_seqChildren[1]->m_scalarString); + else if (node.m_seqChildren.size() == 1) + path = hecl::ProjectPath(m_project.getProjectWorkingPath(), node.m_seqChildren[0]->m_scalarString); + } + if (!path.isFileOrGlob()) + continue; + urde::SObjectTag pathTag = tagFromPath(path, m_backgroundBlender); + if (pathTag) + { + std::unique_lock lk(m_backgroundIndexMutex); + m_catalogNameToTag[pLower] = pathTag; + m_catalogTagToName[pathTag] = p.first; + WriteNameTag(nameWriter, pathTag, p.first); +#if 0 + fprintf(stderr, "%s %s %08X\n", + p.first.c_str(), + pathTag.type.toString().c_str(), uint32_t(pathTag.id)); +#endif + } + } +} + +void SpecBase::backgroundIndexRecursiveCatalogs(const hecl::ProjectPath& dir, + athena::io::YAMLDocWriter& nameWriter, + int level) +{ + hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(), + hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted, + false, false, true); + + /* Enumerate all items */ + for (const hecl::DirectoryEnumerator::Entry& ent : dEnum) + { + hecl::ProjectPath path(dir, ent.m_name); + if (ent.m_isDir && level < 1) + backgroundIndexRecursiveCatalogs(path, nameWriter, level + 1); + else + { + if (!path.isFile()) + continue; + + /* Read catalog.yaml for .pak directory if exists */ + if (level == 1 && !ent.m_name.compare(_S("!catalog.yaml"))) + { + readCatalog(path, nameWriter); + continue; + } + } + + /* bail if cancelled by client */ + if (!m_backgroundRunning) + break; + } +} + +#if DUMP_CACHE_FILL +static void DumpCacheAdd(const urde::SObjectTag& pathTag, const hecl::ProjectPath& path) +{ + fprintf(stderr, "%s %08X %s\n", + pathTag.type.toString().c_str(), uint32_t(pathTag.id.Value()), + path.getRelativePathUTF8().c_str()); +} +#endif + +bool SpecBase::addFileToIndex(const hecl::ProjectPath& path, + athena::io::YAMLDocWriter& cacheWriter) +{ + /* Avoid redundant filesystem access for re-caches */ + if (m_pathToTag.find(path.hash()) != m_pathToTag.cend()) + return true; + + /* Try as glob */ + hecl::ProjectPath asGlob = path.getWithExtension(_S(".*"), true); + if (m_pathToTag.find(asGlob.hash()) != m_pathToTag.cend()) + return true; + + /* Classify intermediate into tag */ + urde::SObjectTag pathTag = buildTagFromPath(path, m_backgroundBlender); + if (pathTag) + { + std::unique_lock lk(m_backgroundIndexMutex); + bool useGlob = false; + + /* Special multi-resource intermediates */ + if (pathTag.type == SBIG('ANCS')) + { + hecl::BlenderConnection& conn = m_backgroundBlender.getBlenderConnection(); + if (!conn.openBlend(path) || conn.getBlendType() != hecl::BlenderConnection::BlendType::Actor) + return false; + + /* Transform tag to glob */ + pathTag = {SBIG('ANCS'), asGlob.hash().val32()}; + useGlob = true; + + hecl::BlenderConnection::DataStream ds = conn.beginData(); + std::vector armatureNames = ds.getArmatureNames(); + std::vector subtypeNames = ds.getSubtypeNames(); + std::vector actionNames = ds.getActionNames(); + + for (const std::string& arm : armatureNames) + { + hecl::SystemStringView sysStr(arm); + hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".CINF")); + urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender); + m_tagToPath[pathTag] = subPath; + m_pathToTag[subPath.hash()] = pathTag; + WriteTag(cacheWriter, pathTag, subPath); +#if DUMP_CACHE_FILL + DumpCacheAdd(pathTag, subPath); +#endif + } + + for (const std::string& sub : subtypeNames) + { + hecl::SystemStringView sysStr(sub); + hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".CSKR")); + urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender); + m_tagToPath[pathTag] = subPath; + m_pathToTag[subPath.hash()] = pathTag; + WriteTag(cacheWriter, pathTag, subPath); +#if DUMP_CACHE_FILL + DumpCacheAdd(pathTag, subPath); +#endif + + std::vector overlayNames = ds.getSubtypeOverlayNames(sub); + for (const auto& overlay : overlayNames) + { + hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S('.') + + overlay + _S(".CSKR")); + urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender); + m_tagToPath[pathTag] = subPath; + m_pathToTag[subPath.hash()] = pathTag; + WriteTag(cacheWriter, pathTag, subPath); +#if DUMP_CACHE_FILL + DumpCacheAdd(pathTag, subPath); +#endif + } + } + + for (const std::string& act : actionNames) + { + hecl::SystemStringView sysStr(act); + hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".ANIM")); + urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender); + m_tagToPath[pathTag] = subPath; + m_pathToTag[subPath.hash()] = pathTag; + WriteTag(cacheWriter, pathTag, subPath); +#if DUMP_CACHE_FILL + DumpCacheAdd(pathTag, subPath); +#endif + } + } + else if (pathTag.type == SBIG('MLVL')) + { + /* Transform tag to glob */ + pathTag = {SBIG('MLVL'), asGlob.hash().val32()}; + useGlob = true; + + hecl::ProjectPath subPath = asGlob.ensureAuxInfo(_S("MAPW")); + urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender); + m_tagToPath[pathTag] = subPath; + m_pathToTag[subPath.hash()] = pathTag; + WriteTag(cacheWriter, pathTag, subPath); +#if DUMP_CACHE_FILL + DumpCacheAdd(pathTag, subPath); +#endif + + subPath = asGlob.ensureAuxInfo(_S("SAVW")); + pathTag = buildTagFromPath(subPath, m_backgroundBlender); + m_tagToPath[pathTag] = subPath; + m_pathToTag[subPath.hash()] = pathTag; + WriteTag(cacheWriter, pathTag, subPath); +#if DUMP_CACHE_FILL + DumpCacheAdd(pathTag, subPath); +#endif + } + else if (pathTag.type == SBIG('AGSC')) + { + /* Transform tag to glob */ + pathTag = {SBIG('AGSC'), asGlob.hash().val32()}; + useGlob = true; + } + else if (pathTag.type == SBIG('MREA')) + { + hecl::ProjectPath subPath = path.ensureAuxInfo(_S("PATH")); + urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender); + m_tagToPath[pathTag] = subPath; + m_pathToTag[subPath.hash()] = pathTag; + WriteTag(cacheWriter, pathTag, subPath); +#if DUMP_CACHE_FILL + DumpCacheAdd(pathTag, subPath); +#endif + } + + /* Cache in-memory */ + const hecl::ProjectPath& usePath = useGlob ? asGlob : path; + m_tagToPath[pathTag] = usePath; + m_pathToTag[usePath.hash()] = pathTag; + WriteTag(cacheWriter, pathTag, usePath); +#if DUMP_CACHE_FILL + DumpCacheAdd(pathTag, usePath); +#endif + } + + return true; +} + +void SpecBase::backgroundIndexRecursiveProc(const hecl::ProjectPath& dir, + athena::io::YAMLDocWriter& cacheWriter, + athena::io::YAMLDocWriter& nameWriter, + int level) +{ + hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(), + hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted, + false, false, true); + + /* Enumerate all items */ + for (const hecl::DirectoryEnumerator::Entry& ent : dEnum) + { + hecl::ProjectPath path(dir, ent.m_name); + if (ent.m_isDir) + backgroundIndexRecursiveProc(path, cacheWriter, nameWriter, level + 1); + else + { + if (!path.isFile()) + continue; + + /* Read catalog.yaml for .pak directory if exists */ + if (level == 1 && !ent.m_name.compare(_S("!catalog.yaml"))) + { + readCatalog(path, nameWriter); + continue; + } + + /* Index the regular file */ + addFileToIndex(path, cacheWriter); + } + + /* bail if cancelled by client */ + if (!m_backgroundRunning) + break; + } +} + +void SpecBase::backgroundIndexProc() +{ + logvisor::RegisterThreadName("Resource Index Thread"); + + hecl::ProjectPath tagCachePath(m_project.getProjectCookedPath(getOriginalSpec()), _S("tag_cache.yaml")); + hecl::ProjectPath nameCachePath(m_project.getProjectCookedPath(getOriginalSpec()), _S("name_cache.yaml")); + hecl::ProjectPath specRoot(m_project.getProjectWorkingPath(), getOriginalSpec().m_name); + + /* Cache will be overwritten with validated entries afterwards */ + athena::io::YAMLDocWriter cacheWriter(nullptr); + athena::io::YAMLDocWriter nameWriter(nullptr); + + /* Read in tag cache */ + if (tagCachePath.isFile()) + { + athena::io::FileReader reader(tagCachePath.getAbsolutePath()); + if (reader.isOpen()) + { + Log.report(logvisor::Info, _S("Cache index of '%s' loading"), getOriginalSpec().m_name); + athena::io::YAMLDocReader cacheReader; + if (cacheReader.parse(&reader)) + { + std::unique_lock lk(m_backgroundIndexMutex); + size_t tagCount = cacheReader.getRootNode()->m_mapChildren.size(); + m_tagToPath.reserve(tagCount); + m_pathToTag.reserve(tagCount); + size_t loadIdx = 0; + for (const auto& child : cacheReader.getRootNode()->m_mapChildren) + { + const athena::io::YAMLNode& node = *child.second; + unsigned long id = strtoul(child.first.c_str(), nullptr, 16); + hecl::FourCC type(node.m_seqChildren.at(0)->m_scalarString.c_str()); + hecl::ProjectPath path(m_project.getProjectWorkingPath(), + node.m_seqChildren.at(1)->m_scalarString); + + if (path.isFileOrGlob()) + { + urde::SObjectTag pathTag(type, id); + m_tagToPath[pathTag] = path; + m_pathToTag[path.hash()] = pathTag; + WriteTag(cacheWriter, pathTag, path); + } + fprintf(stderr, "\r %" PRISize " / %" PRISize, ++loadIdx, tagCount); + } + fprintf(stderr, "\n"); + } + Log.report(logvisor::Info, _S("Cache index of '%s' loaded; %d tags"), + getOriginalSpec().m_name, m_tagToPath.size()); + + if (nameCachePath.isFile()) + { + /* Read in name cache */ + Log.report(logvisor::Info, _S("Name index of '%s' loading"), getOriginalSpec().m_name); + athena::io::FileReader nreader(nameCachePath.getAbsolutePath()); + athena::io::YAMLDocReader nameReader; + if (nameReader.parse(&nreader)) + { + std::unique_lock lk(m_backgroundIndexMutex); + m_catalogNameToTag.reserve(nameReader.getRootNode()->m_mapChildren.size()); + m_catalogTagToName.reserve(nameReader.getRootNode()->m_mapChildren.size()); + for (const auto& child : nameReader.getRootNode()->m_mapChildren) + { + unsigned long id = strtoul(child.second->m_scalarString.c_str(), nullptr, 16); + auto search = m_tagToPath.find(urde::SObjectTag(FourCC(), uint32_t(id))); + if (search != m_tagToPath.cend()) + { + std::string chLower = child.first; + std::transform(chLower.cbegin(), chLower.cend(), chLower.begin(), tolower); + m_catalogNameToTag[chLower] = search->first; + m_catalogTagToName[search->first] = child.first; + WriteNameTag(nameWriter, search->first, child.first); + } + } + } + Log.report(logvisor::Info, _S("Name index of '%s' loaded; %d names"), + getOriginalSpec().m_name, m_catalogNameToTag.size()); + } + } + } + + /* Add special original IDs resource if exists (not name-cached to disk) */ + hecl::ProjectPath oidsPath(specRoot, "!original_ids.yaml"); + urde::SObjectTag oidsTag = buildTagFromPath(oidsPath, m_backgroundBlender); + if (oidsTag) + { + m_catalogNameToTag["mp1originalids"] = oidsTag; + m_catalogTagToName[oidsTag] = "MP1OriginalIDs"; + } + + Log.report(logvisor::Info, _S("Background index of '%s' started"), getOriginalSpec().m_name); + backgroundIndexRecursiveProc(specRoot, cacheWriter, nameWriter, 0); + + tagCachePath.makeDirChain(false); + athena::io::FileWriter twriter(tagCachePath.getAbsolutePath()); + cacheWriter.finish(&twriter); + + athena::io::FileWriter nwriter(nameCachePath.getAbsolutePath()); + nameWriter.finish(&nwriter); + + m_backgroundBlender.shutdown(); + Log.report(logvisor::Info, _S("Background index of '%s' complete; %d tags, %d names"), + getOriginalSpec().m_name, m_tagToPath.size(), m_catalogNameToTag.size()); + m_backgroundRunning = false; +} + +void SpecBase::cancelBackgroundIndex() +{ + m_backgroundRunning = false; + if (m_backgroundIndexTh.joinable()) + m_backgroundIndexTh.join(); +} + +void SpecBase::beginBackgroundIndex() +{ + cancelBackgroundIndex(); + clearTagCache(); + m_backgroundRunning = true; + m_backgroundIndexTh = std::thread(std::bind(&SpecBase::backgroundIndexProc, this)); +} + +void SpecBase::waitForIndexComplete() const +{ + std::unique_lock lk(const_cast(*this).m_backgroundIndexMutex); + while (m_backgroundRunning) + { + lk.unlock(); + std::this_thread::sleep_for(std::chrono::milliseconds(2)); + lk.lock(); + } +} + } diff --git a/DataSpec/SpecBase.hpp b/DataSpec/SpecBase.hpp index 2c93382dd..41801a491 100644 --- a/DataSpec/SpecBase.hpp +++ b/DataSpec/SpecBase.hpp @@ -2,15 +2,14 @@ #define SPECBASE_HPP #include +#include #include #include +#include +#include #include "hecl/Blender/BlenderConnection.hpp" - -namespace urde -{ -struct SObjectTag; -} +#include "Runtime/RetroTypes.hpp" namespace DataSpec { @@ -29,8 +28,9 @@ struct SpecBase : hecl::Database::IDataSpec void doCook(const hecl::ProjectPath& path, const hecl::ProjectPath& cookedPath, bool fast, hecl::BlenderToken& btok, FCookProgress progress); - bool canPackage(const PackagePassInfo& info); - void doPackage(const PackagePassInfo& info); + bool canPackage(const hecl::ProjectPath& path); + void doPackage(const hecl::ProjectPath& path, const hecl::Database::DataSpecEntry* entry, + bool fast, hecl::BlenderToken& btok, FProgress progress, hecl::ClientProcess* cp); /* Extract handlers */ virtual bool checkStandaloneID(const char* id) const=0; @@ -46,7 +46,7 @@ struct SpecBase : hecl::Database::IDataSpec FProgress progress)=0; /* Convert path to object tag */ - virtual urde::SObjectTag BuildTagFromPath(const hecl::ProjectPath& path, + virtual urde::SObjectTag buildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const=0; /* Even if PC spec is being cooked, this will return the vanilla GCN spec */ @@ -107,23 +107,93 @@ struct SpecBase : hecl::Database::IDataSpec void flattenDependencies(const class UniqueID64& id, std::vector& pathsOut); virtual void flattenDependenciesYAML(athena::io::IStreamReader& fin, std::vector& pathsOut)=0; + virtual void buildWorldPakList(const hecl::ProjectPath& worldPath, + const hecl::ProjectPath& worldPathCooked, + hecl::BlenderToken& btok, + athena::io::FileWriter& w, + std::vector& listOut, + atUint64& resTableOffset) {} + virtual void buildPakList(hecl::BlenderToken& btok, + athena::io::FileWriter& w, + const std::vector& list, + const std::vector>& nameList, + atUint64& resTableOffset) {} + virtual void writePakFileIndex(athena::io::FileWriter& w, + const std::vector& tags, + const std::vector>& index, + atUint64 resTableOffset) {} + virtual std::pair, size_t> + compressPakData(const urde::SObjectTag& tag, const uint8_t* data, size_t len) { return {}; } + const hecl::ProjectPath& getMasterShaderPath() const {return m_masterShader;} /* Support functions for resolving paths from IDs */ virtual hecl::ProjectPath getWorking(class UniqueID32&) {return hecl::ProjectPath();} virtual hecl::ProjectPath getWorking(class UniqueID64&) {return hecl::ProjectPath();} + hecl::ProjectPath getCookedPath(const hecl::ProjectPath& working, bool pcTarget) const; + /* Project accessor */ hecl::Database::Project& getProject() const {return m_project;} /* Extract RandomStatic entropy */ - void ExtractRandomStaticEntropy(const uint8_t* buf, const hecl::ProjectPath& noAramPath); + void extractRandomStaticEntropy(const uint8_t* buf, const hecl::ProjectPath& noAramPath); + + /* Tag cache functions */ + urde::SObjectTag tagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const; + hecl::ProjectPath pathFromTag(const urde::SObjectTag& tag) const; + bool waitForTagReady(const urde::SObjectTag& tag, const hecl::ProjectPath*& pathOut); + const urde::SObjectTag* getResourceIdByName(const char* name) const; + hecl::FourCC getResourceTypeById(urde::CAssetId id) const; + void enumerateResources(const std::function& lambda) const; + void enumerateNamedResources( + const std::function& lambda) const; + void cancelBackgroundIndex(); + void beginBackgroundIndex(); + bool backgroundIndexRunning() const { return m_backgroundRunning; } + void waitForIndexComplete() const; + + virtual void getTagListForFile(const char* pakName, std::vector& out) const {} SpecBase(const hecl::Database::DataSpecEntry* specEntry, hecl::Database::Project& project, bool pc); + ~SpecBase(); protected: hecl::Database::Project& m_project; bool m_pc; hecl::ProjectPath m_masterShader; + + std::unordered_map m_tagToPath; + std::unordered_map m_pathToTag; + std::unordered_map m_catalogNameToTag; + std::unordered_map m_catalogTagToName; + void clearTagCache(); + + hecl::BlenderToken m_backgroundBlender; + std::thread m_backgroundIndexTh; + std::mutex m_backgroundIndexMutex; + bool m_backgroundRunning = false; + + void readCatalog(const hecl::ProjectPath& catalogPath, + athena::io::YAMLDocWriter& nameWriter); + bool addFileToIndex(const hecl::ProjectPath& path, + athena::io::YAMLDocWriter& cacheWriter); + void backgroundIndexRecursiveProc(const hecl::ProjectPath& path, + athena::io::YAMLDocWriter& cacheWriter, + athena::io::YAMLDocWriter& nameWriter, + int level); + void backgroundIndexRecursiveCatalogs(const hecl::ProjectPath& path, + athena::io::YAMLDocWriter& nameWriter, + int level); + void backgroundIndexProc(); + + void recursiveBuildResourceList(std::vector& listOut, + const hecl::ProjectPath& path, + hecl::BlenderToken& btok); + void copyBuildListData(std::vector>& fileIndex, + const std::vector& buildList, + const hecl::Database::DataSpecEntry* entry, + bool fast, FProgress progress, athena::io::FileWriter& pakOut); + private: std::unique_ptr m_disc; bool m_isWii; diff --git a/DataSpec/SpecMP1.cpp b/DataSpec/SpecMP1.cpp index b1a943a09..78ef02b0f 100644 --- a/DataSpec/SpecMP1.cpp +++ b/DataSpec/SpecMP1.cpp @@ -430,7 +430,7 @@ struct SpecMP1 : SpecBase /* Extract part of .dol for RandomStatic entropy */ hecl::ProjectPath noAramPath(m_project.getProjectWorkingPath(), _S("MP1/NoARAM")); - ExtractRandomStaticEntropy(m_dolBuf.get() + 0x4f60, noAramPath); + extractRandomStaticEntropy(m_dolBuf.get() + 0x4f60, noAramPath); return true; } @@ -517,7 +517,7 @@ struct SpecMP1 : SpecBase }); } - urde::SObjectTag BuildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const + urde::SObjectTag buildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const { if (hecl::StringUtils::EndsWith(path.getAuxInfo(), _S(".CINF"))) return {SBIG('CINF'), path.hash().val32()}; @@ -718,6 +718,18 @@ struct SpecMP1 : SpecBase return {}; } + void getTagListForFile(const char* pakName, std::vector& out) const + { + std::string pathPrefix("MP1/"); + pathPrefix += pakName; + pathPrefix += '/'; + + std::unique_lock lk(const_cast(*this).m_backgroundIndexMutex); + for (const auto& tag : m_tagToPath) + if (!tag.second.getRelativePathUTF8().compare(0, pathPrefix.size(), pathPrefix)) + out.push_back(tag.first); + } + void cookMesh(const hecl::ProjectPath& out, const hecl::ProjectPath& in, BlendStream& ds, bool fast, hecl::BlenderToken& btok, FCookProgress progress) { @@ -1109,6 +1121,210 @@ struct SpecMP1 : SpecBase } } + void buildWorldPakList(const hecl::ProjectPath& worldPath, + const hecl::ProjectPath& worldPathCooked, + hecl::BlenderToken& btok, + athena::io::FileWriter& w, + std::vector& listOut, + atUint64& resTableOffset) + { + DNAMP1::MLVL mlvl; + { + athena::io::FileReader r(worldPathCooked.getAbsolutePath()); + if (r.hasError()) + Log.report(logvisor::Fatal, _S("Unable to open world %s"), worldPathCooked.getRelativePath().c_str()); + mlvl.read(r); + } + + size_t count = 5; + for (const auto& area : mlvl.areas) + for (const auto& dep : area.deps) + ++count; + listOut.reserve(count); + + urde::SObjectTag worldTag = tagFromPath(worldPath.getWithExtension(_S(".*"), true), btok); + + w.writeUint32Big(0x80030005); + w.writeUint32Big(0); + + w.writeUint32Big(1); + DNAMP1::PAK::NameEntry nameEnt; + hecl::ProjectPath parentDir = worldPath.getParentPath(); + nameEnt.type = worldTag.type; + nameEnt.id = atUint32(worldTag.id.Value()); + nameEnt.nameLen = atUint32(hecl::StrLen(parentDir.getLastComponent())); + nameEnt.name = parentDir.getLastComponent(); + nameEnt.write(w); + + w.writeUint32Big(atUint32(count)); + resTableOffset = w.position(); + for (const auto& area : mlvl.areas) + for (const auto& dep : area.deps) + listOut.push_back({dep.type, dep.id.toUint32()}); + + urde::SObjectTag nameTag(FOURCC('STRG'), mlvl.worldNameId.toUint32()); + if (nameTag) + listOut.push_back(nameTag); + + urde::SObjectTag savwTag(FOURCC('SAVW'), mlvl.saveWorldId.toUint32()); + if (savwTag) + { + if (hecl::ProjectPath savwPath = pathFromTag(savwTag)) + m_project.cookPath(savwPath, {}, false, true); + listOut.push_back(savwTag); + } + + urde::SObjectTag mapTag(FOURCC('MAPW'), mlvl.worldMap.toUint32()); + if (mapTag) + { + if (hecl::ProjectPath mapPath = pathFromTag(mapTag)) + { + m_project.cookPath(mapPath, {}, false, true); + if (hecl::ProjectPath mapCookedPath = getCookedPath(mapPath, true)) + { + athena::io::FileReader r(mapCookedPath.getAbsolutePath()); + if (r.hasError()) + Log.report(logvisor::Fatal, _S("Unable to open %s"), mapCookedPath.getRelativePath().c_str()); + + if (r.readUint32Big() != 0xDEADF00D) + Log.report(logvisor::Fatal, _S("Corrupt MAPW %s"), mapCookedPath.getRelativePath().c_str()); + r.readUint32Big(); + atUint32 mapaCount = r.readUint32Big(); + for (int i=0 ; i textures = data.getTextures(); + for (const auto& tex : textures) + { + urde::SObjectTag texTag = tagFromPath(tex, btok); + if (!texTag) + Log.report(logvisor::Fatal, _S("Unable to resolve %s"), tex.getRelativePath().c_str()); + listOut.push_back(texTag); + } + } + listOut.push_back(skyboxTag); + } + + listOut.push_back(worldTag); + + for (const auto& item : listOut) + { + DNAMP1::PAK::Entry ent; + ent.compressed = 0; + ent.type = item.type; + ent.id = atUint32(item.id.Value()); + ent.size = 0; + ent.offset = 0; + ent.write(w); + } + } + + void buildPakList(hecl::BlenderToken& btok, + athena::io::FileWriter& w, + const std::vector& list, + const std::vector>& nameList, + atUint64& resTableOffset) + { + w.writeUint32Big(0x80030005); + w.writeUint32Big(0); + + w.writeUint32Big(atUint32(nameList.size())); + for (const auto& item : nameList) + { + DNAMP1::PAK::NameEntry nameEnt; + nameEnt.type = item.first.type; + nameEnt.id = atUint32(item.first.id.Value()); + nameEnt.nameLen = atUint32(item.second.size()); + nameEnt.name = item.second; + nameEnt.write(w); + } + + w.writeUint32Big(atUint32(list.size())); + resTableOffset = w.position(); + for (const auto& item : list) + { + DNAMP1::PAK::Entry ent; + ent.compressed = 0; + ent.type = item.type; + ent.id = atUint32(item.id.Value()); + ent.size = 0; + ent.offset = 0; + ent.write(w); + } + } + + void writePakFileIndex(athena::io::FileWriter& w, + const std::vector& tags, + const std::vector>& index, + atUint64 resTableOffset) + { + w.seek(resTableOffset, athena::Begin); + + auto it = tags.begin(); + for (const auto& item : index) + { + const urde::SObjectTag& tag = *it++; + DNAMP1::PAK::Entry ent; + ent.compressed = atUint32(std::get<2>(item)); + ent.type = tag.type; + ent.id = atUint32(tag.id.Value()); + ent.size = atUint32(std::get<1>(item)); + ent.offset = atUint32(std::get<0>(item)); + ent.write(w); + } + } + + std::pair, size_t> + compressPakData(const urde::SObjectTag& tag, const uint8_t* data, size_t len) + { + bool doCompress = false; + switch (tag.type) + { + case SBIG('TXTR'): + case SBIG('CMDL'): + case SBIG('CSKR'): + case SBIG('ANCS'): + case SBIG('ANIM'): + case SBIG('FONT'): + doCompress = true; + break; + case SBIG('PART'): + case SBIG('ELSC'): + case SBIG('SWHC'): + case SBIG('WPSC'): + case SBIG('DPSC'): + case SBIG('CRSC'): + doCompress = len >= 0x400; + break; + default: + break; + } + if (!doCompress) + return {}; + + uLong destLen = compressBound(len); + std::pair, size_t> ret; + ret.first.reset(new uint8_t[destLen]); + compress(ret.first.get(), &destLen, data, len); + ret.second = destLen; + return ret; + }; + void cookAudioGroup(const hecl::ProjectPath& out, const hecl::ProjectPath& in, FCookProgress progress) { DNAMP1::AGSC::Cook(in, out); diff --git a/DataSpec/SpecMP2.cpp b/DataSpec/SpecMP2.cpp index f9cddac9d..ff00f6492 100644 --- a/DataSpec/SpecMP2.cpp +++ b/DataSpec/SpecMP2.cpp @@ -318,7 +318,7 @@ struct SpecMP2 : SpecBase }); } - urde::SObjectTag BuildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const + urde::SObjectTag buildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const { return {}; } diff --git a/DataSpec/SpecMP3.cpp b/DataSpec/SpecMP3.cpp index b9e52f66a..e98a01b03 100644 --- a/DataSpec/SpecMP3.cpp +++ b/DataSpec/SpecMP3.cpp @@ -512,7 +512,7 @@ struct SpecMP3 : SpecBase return false; } - urde::SObjectTag BuildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const + urde::SObjectTag buildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const { return {}; } diff --git a/Editor/ProjectResourceFactoryBase.cpp b/Editor/ProjectResourceFactoryBase.cpp index 90a76c186..211136dbb 100644 --- a/Editor/ProjectResourceFactoryBase.cpp +++ b/Editor/ProjectResourceFactoryBase.cpp @@ -1,445 +1,21 @@ #include "ProjectResourceFactoryBase.hpp" #include "Runtime/IObj.hpp" -#define DUMP_CACHE_FILL 1 - namespace urde { static logvisor::Module Log("urde::ProjectResourceFactoryBase"); -static void WriteTag(athena::io::YAMLDocWriter& cacheWriter, - const SObjectTag& pathTag, const hecl::ProjectPath& path) -{ - char idStr[9]; - snprintf(idStr, 9, "%08X", uint32_t(pathTag.id.Value())); - if (auto v = cacheWriter.enterSubVector(idStr)) - { - cacheWriter.writeString(nullptr, pathTag.type.toString().c_str()); - cacheWriter.writeString(nullptr, path.getAuxInfo().size() ? - (path.getRelativePathUTF8() + '|' + path.getAuxInfoUTF8()) : - path.getRelativePathUTF8()); - } -} - -static void WriteNameTag(athena::io::YAMLDocWriter& nameWriter, - const SObjectTag& pathTag, - const std::string& name) -{ - char idStr[9]; - snprintf(idStr, 9, "%08X", uint32_t(pathTag.id.Value())); - nameWriter.writeString(name.c_str(), idStr); -} - -void ProjectResourceFactoryBase::Clear() -{ - m_tagToPath.clear(); - m_pathToTag.clear(); - m_catalogNameToTag.clear(); -} - -SObjectTag ProjectResourceFactoryBase::TagFromPath(const hecl::ProjectPath& path, - hecl::BlenderToken& btok) const -{ - auto search = m_pathToTag.find(path.hash()); - if (search != m_pathToTag.cend()) - return search->second; - return BuildTagFromPath(path, btok); -} - -void ProjectResourceFactoryBase::ReadCatalog(const hecl::ProjectPath& catalogPath, - athena::io::YAMLDocWriter& nameWriter) -{ - athena::io::FileReader freader(catalogPath.getAbsolutePath()); - if (!freader.isOpen()) - return; - - athena::io::YAMLDocReader reader; - bool res = reader.parse(&freader); - if (!res) - return; - - const athena::io::YAMLNode* root = reader.getRootNode(); - for (const auto& p : root->m_mapChildren) - { - /* Hash as lowercase since lookup is case-insensitive */ - std::string pLower = p.first; - std::transform(pLower.cbegin(), pLower.cend(), pLower.begin(), tolower); - - /* Avoid redundant filesystem access for re-caches */ - if (m_catalogNameToTag.find(pLower) != m_catalogNameToTag.cend()) - continue; - - athena::io::YAMLNode& node = *p.second; - hecl::ProjectPath path; - if (node.m_type == YAML_SCALAR_NODE) - { - path = hecl::ProjectPath(m_proj->getProjectWorkingPath(), node.m_scalarString); - } - else if (node.m_type == YAML_SEQUENCE_NODE) - { - if (node.m_seqChildren.size() >= 2) - path = hecl::ProjectPath(m_proj->getProjectWorkingPath(), node.m_seqChildren[0]->m_scalarString). - ensureAuxInfo(node.m_seqChildren[1]->m_scalarString); - else if (node.m_seqChildren.size() == 1) - path = hecl::ProjectPath(m_proj->getProjectWorkingPath(), node.m_seqChildren[0]->m_scalarString); - } - if (!path.isFileOrGlob()) - continue; - SObjectTag pathTag = TagFromPath(path, m_backgroundBlender); - if (pathTag) - { - std::unique_lock lk(m_backgroundIndexMutex); - m_catalogNameToTag[pLower] = pathTag; - WriteNameTag(nameWriter, pathTag, p.first); -#if 0 - fprintf(stderr, "%s %s %08X\n", - p.first.c_str(), - pathTag.type.toString().c_str(), uint32_t(pathTag.id)); -#endif - } - } -} - -void ProjectResourceFactoryBase::BackgroundIndexRecursiveCatalogs(const hecl::ProjectPath& dir, - athena::io::YAMLDocWriter& nameWriter, - int level) -{ - hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(), - hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted, - false, false, true); - - /* Enumerate all items */ - for (const hecl::DirectoryEnumerator::Entry& ent : dEnum) - { - hecl::ProjectPath path(dir, ent.m_name); - if (ent.m_isDir && level < 1) - BackgroundIndexRecursiveCatalogs(path, nameWriter, level+1); - else - { - if (!path.isFile()) - continue; - - /* Read catalog.yaml for .pak directory if exists */ - if (level == 1 && !ent.m_name.compare(_S("!catalog.yaml"))) - { - ReadCatalog(path, nameWriter); - continue; - } - } - - /* bail if cancelled by client */ - if (!m_backgroundRunning) - break; - } -} - -#if DUMP_CACHE_FILL -static void DumpCacheAdd(const SObjectTag& pathTag, const hecl::ProjectPath& path) -{ - fprintf(stderr, "%s %08X %s\n", - pathTag.type.toString().c_str(), uint32_t(pathTag.id.Value()), - path.getRelativePathUTF8().c_str()); -} -#endif - -bool ProjectResourceFactoryBase::AddFileToIndex(const hecl::ProjectPath& path, - athena::io::YAMLDocWriter& cacheWriter) -{ - /* Avoid redundant filesystem access for re-caches */ - if (m_pathToTag.find(path.hash()) != m_pathToTag.cend()) - return true; - - /* Try as glob */ - hecl::ProjectPath asGlob = path.getWithExtension(_S(".*"), true); - if (m_pathToTag.find(asGlob.hash()) != m_pathToTag.cend()) - return true; - - /* Classify intermediate into tag */ - SObjectTag pathTag = BuildTagFromPath(path, m_backgroundBlender); - if (pathTag) - { - std::unique_lock lk(m_backgroundIndexMutex); - bool useGlob = false; - - /* Special multi-resource intermediates */ - if (pathTag.type == SBIG('ANCS')) - { - hecl::BlenderConnection& conn = m_backgroundBlender.getBlenderConnection(); - if (!conn.openBlend(path) || conn.getBlendType() != hecl::BlenderConnection::BlendType::Actor) - return false; - - /* Transform tag to glob */ - pathTag = {SBIG('ANCS'), asGlob.hash().val32()}; - useGlob = true; - - hecl::BlenderConnection::DataStream ds = conn.beginData(); - std::vector armatureNames = ds.getArmatureNames(); - std::vector subtypeNames = ds.getSubtypeNames(); - std::vector actionNames = ds.getActionNames(); - - for (const std::string& arm : armatureNames) - { - hecl::SystemStringView sysStr(arm); - hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".CINF")); - SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender); - m_tagToPath[pathTag] = subPath; - m_pathToTag[subPath.hash()] = pathTag; - WriteTag(cacheWriter, pathTag, subPath); -#if DUMP_CACHE_FILL - DumpCacheAdd(pathTag, subPath); -#endif - } - - for (const std::string& sub : subtypeNames) - { - hecl::SystemStringView sysStr(sub); - hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".CSKR")); - SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender); - m_tagToPath[pathTag] = subPath; - m_pathToTag[subPath.hash()] = pathTag; - WriteTag(cacheWriter, pathTag, subPath); -#if DUMP_CACHE_FILL - DumpCacheAdd(pathTag, subPath); -#endif - } - - for (const std::string& act : actionNames) - { - hecl::SystemStringView sysStr(act); - hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".ANIM")); - SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender); - m_tagToPath[pathTag] = subPath; - m_pathToTag[subPath.hash()] = pathTag; - WriteTag(cacheWriter, pathTag, subPath); -#if DUMP_CACHE_FILL - DumpCacheAdd(pathTag, subPath); -#endif - } - } - else if (pathTag.type == SBIG('MLVL')) - { - /* Transform tag to glob */ - pathTag = {SBIG('MLVL'), asGlob.hash().val32()}; - useGlob = true; - - hecl::ProjectPath subPath = asGlob.ensureAuxInfo(_S("MAPW")); - SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender); - m_tagToPath[pathTag] = subPath; - m_pathToTag[subPath.hash()] = pathTag; - WriteTag(cacheWriter, pathTag, subPath); -#if DUMP_CACHE_FILL - DumpCacheAdd(pathTag, subPath); -#endif - - subPath = asGlob.ensureAuxInfo(_S("SAVW")); - pathTag = BuildTagFromPath(subPath, m_backgroundBlender); - m_tagToPath[pathTag] = subPath; - m_pathToTag[subPath.hash()] = pathTag; - WriteTag(cacheWriter, pathTag, subPath); -#if DUMP_CACHE_FILL - DumpCacheAdd(pathTag, subPath); -#endif - } - else if (pathTag.type == SBIG('AGSC')) - { - /* Transform tag to glob */ - pathTag = {SBIG('AGSC'), asGlob.hash().val32()}; - useGlob = true; - } - else if (pathTag.type == SBIG('MREA')) - { - hecl::ProjectPath subPath = path.ensureAuxInfo(_S("PATH")); - SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender); - m_tagToPath[pathTag] = subPath; - m_pathToTag[subPath.hash()] = pathTag; - WriteTag(cacheWriter, pathTag, subPath); -#if DUMP_CACHE_FILL - DumpCacheAdd(pathTag, subPath); -#endif - } - - /* Cache in-memory */ - const hecl::ProjectPath& usePath = useGlob ? asGlob : path; - m_tagToPath[pathTag] = usePath; - m_pathToTag[usePath.hash()] = pathTag; - WriteTag(cacheWriter, pathTag, usePath); -#if DUMP_CACHE_FILL - DumpCacheAdd(pathTag, usePath); -#endif - } - - return true; -} - -void ProjectResourceFactoryBase::BackgroundIndexRecursiveProc(const hecl::ProjectPath& dir, - athena::io::YAMLDocWriter& cacheWriter, - athena::io::YAMLDocWriter& nameWriter, - int level) -{ - hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(), - hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted, - false, false, true); - - /* Enumerate all items */ - for (const hecl::DirectoryEnumerator::Entry& ent : dEnum) - { - hecl::ProjectPath path(dir, ent.m_name); - if (ent.m_isDir) - BackgroundIndexRecursiveProc(path, cacheWriter, nameWriter, level+1); - else - { - if (!path.isFile()) - continue; - - /* Read catalog.yaml for .pak directory if exists */ - if (level == 1 && !ent.m_name.compare(_S("!catalog.yaml"))) - { - ReadCatalog(path, nameWriter); - continue; - } - - /* Index the regular file */ - AddFileToIndex(path, cacheWriter); - } - - /* bail if cancelled by client */ - if (!m_backgroundRunning) - break; - } -} - -void ProjectResourceFactoryBase::BackgroundIndexProc() -{ - logvisor::RegisterThreadName("Resource Index Thread"); - - hecl::ProjectPath tagCachePath(m_proj->getProjectCookedPath(*m_origSpec), _S("tag_cache.yaml")); - hecl::ProjectPath nameCachePath(m_proj->getProjectCookedPath(*m_origSpec), _S("name_cache.yaml")); - hecl::ProjectPath specRoot(m_proj->getProjectWorkingPath(), m_origSpec->m_name); - - /* Cache will be overwritten with validated entries afterwards */ - athena::io::YAMLDocWriter cacheWriter(nullptr); - athena::io::YAMLDocWriter nameWriter(nullptr); - - /* Read in tag cache */ - if (tagCachePath.isFile()) - { - athena::io::FileReader reader(tagCachePath.getAbsolutePath()); - if (reader.isOpen()) - { - Log.report(logvisor::Info, _S("Cache index of '%s' loading"), m_origSpec->m_name); - athena::io::YAMLDocReader cacheReader; - if (cacheReader.parse(&reader)) - { - std::unique_lock lk(m_backgroundIndexMutex); - size_t tagCount = cacheReader.getRootNode()->m_mapChildren.size(); - m_tagToPath.reserve(tagCount); - m_pathToTag.reserve(tagCount); - size_t loadIdx = 0; - for (const auto& child : cacheReader.getRootNode()->m_mapChildren) - { - const athena::io::YAMLNode& node = *child.second; - unsigned long id = strtoul(child.first.c_str(), nullptr, 16); - hecl::FourCC type(node.m_seqChildren.at(0)->m_scalarString.c_str()); - hecl::ProjectPath path(m_proj->getProjectWorkingPath(), - node.m_seqChildren.at(1)->m_scalarString); - - if (path.isFileOrGlob()) - { - SObjectTag pathTag(type, id); - m_tagToPath[pathTag] = path; - m_pathToTag[path.hash()] = pathTag; - WriteTag(cacheWriter, pathTag, path); - } - fprintf(stderr, "\r %" PRISize " / %" PRISize, ++loadIdx, tagCount); - } - fprintf(stderr, "\n"); - } - Log.report(logvisor::Info, _S("Cache index of '%s' loaded; %d tags"), - m_origSpec->m_name, m_tagToPath.size()); - - if (nameCachePath.isFile()) - { - /* Read in name cache */ - Log.report(logvisor::Info, _S("Name index of '%s' loading"), m_origSpec->m_name); - athena::io::FileReader nreader(nameCachePath.getAbsolutePath()); - athena::io::YAMLDocReader nameReader; - if (nameReader.parse(&nreader)) - { - std::unique_lock lk(m_backgroundIndexMutex); - m_catalogNameToTag.reserve(nameReader.getRootNode()->m_mapChildren.size()); - for (const auto& child : nameReader.getRootNode()->m_mapChildren) - { - unsigned long id = strtoul(child.second->m_scalarString.c_str(), nullptr, 16); - auto search = m_tagToPath.find(SObjectTag(FourCC(), uint32_t(id))); - if (search != m_tagToPath.cend()) - { - std::string chLower = child.first; - std::transform(chLower.cbegin(), chLower.cend(), chLower.begin(), tolower); - m_catalogNameToTag[chLower] = search->first; - WriteNameTag(nameWriter, search->first, child.first); - } - } - } - Log.report(logvisor::Info, _S("Name index of '%s' loaded; %d names"), - m_origSpec->m_name, m_catalogNameToTag.size()); - } - } - } - - /* Add special original IDs resource if exists (not name-cached to disk) */ - hecl::ProjectPath oidsPath(specRoot, "!original_ids.yaml"); - SObjectTag oidsTag = BuildTagFromPath(oidsPath, m_backgroundBlender); - if (oidsTag) - m_catalogNameToTag["mp1originalids"] = oidsTag; - - Log.report(logvisor::Info, _S("Background index of '%s' started"), m_origSpec->m_name); - BackgroundIndexRecursiveProc(specRoot, cacheWriter, nameWriter, 0); - - tagCachePath.makeDirChain(false); - athena::io::FileWriter twriter(tagCachePath.getAbsolutePath()); - cacheWriter.finish(&twriter); - - athena::io::FileWriter nwriter(nameCachePath.getAbsolutePath()); - nameWriter.finish(&nwriter); - - m_backgroundBlender.shutdown(); - Log.report(logvisor::Info, _S("Background index of '%s' complete; %d tags, %d names"), - m_origSpec->m_name, m_tagToPath.size(), m_catalogNameToTag.size()); - m_backgroundRunning = false; -} - -void ProjectResourceFactoryBase::CancelBackgroundIndex() -{ - m_backgroundRunning = false; - if (m_backgroundIndexTh.joinable()) - m_backgroundIndexTh.join(); -} - void ProjectResourceFactoryBase::BeginBackgroundIndex (hecl::Database::Project& proj, const hecl::Database::DataSpecEntry& origSpec, const hecl::Database::DataSpecEntry& pcSpec) { CancelBackgroundIndex(); - Clear(); m_proj = &proj; m_origSpec = &origSpec; m_pcSpec = &pcSpec; m_cookSpec.reset(pcSpec.m_factory(proj, hecl::Database::DataSpecTool::Cook)); - m_backgroundRunning = true; - m_backgroundIndexTh = - std::thread(std::bind(&ProjectResourceFactoryBase::BackgroundIndexProc, this)); -} - -hecl::ProjectPath ProjectResourceFactoryBase::GetCookedPath(const hecl::ProjectPath& working, - bool pcTarget) const -{ - const hecl::Database::DataSpecEntry* spec = m_origSpec; - if (pcTarget) - spec = m_cookSpec->overrideDataSpec(working, m_pcSpec, hecl::SharedBlenderToken); - if (!spec) - return {}; - return working.getCookedPath(*spec); + return static_cast(*m_cookSpec).beginBackgroundIndex(); } bool ProjectResourceFactoryBase::SyncCook(const hecl::ProjectPath& working) @@ -629,35 +205,6 @@ ProjectResourceFactoryBase::_RemoveTask(const SObjectTag& tag) return _RemoveTask(m_asyncLoadMap.find(tag)); }; -bool ProjectResourceFactoryBase::WaitForTagReady(const urde::SObjectTag& tag, - const hecl::ProjectPath*& pathOut) -{ - std::unique_lock lk(m_backgroundIndexMutex); - auto search = m_tagToPath.find(tag); - if (search == m_tagToPath.end()) - { - if (m_backgroundRunning) - { - while (m_backgroundRunning) - { - lk.unlock(); - std::this_thread::sleep_for(std::chrono::milliseconds(2)); - lk.lock(); - search = m_tagToPath.find(tag); - if (search != m_tagToPath.end()) - break; - } - if (search == m_tagToPath.end()) - return false; - } - else - return false; - } - lk.unlock(); - pathOut = &search->second; - return true; -} - bool ProjectResourceFactoryBase::PrepForReadSync(const SObjectTag& tag, const hecl::ProjectPath& path, @@ -922,108 +469,34 @@ bool ProjectResourceFactoryBase::CanBuild(const urde::SObjectTag& tag) const urde::SObjectTag* ProjectResourceFactoryBase::GetResourceIdByName(const char* name) const { - std::string lower = name; - std::transform(lower.cbegin(), lower.cend(), lower.begin(), tolower); - - std::unique_lock lk(const_cast(this)->m_backgroundIndexMutex); - auto search = m_catalogNameToTag.find(lower); - if (search == m_catalogNameToTag.end()) - { - if (m_backgroundRunning) - { - while (m_backgroundRunning) - { - lk.unlock(); - std::this_thread::sleep_for(std::chrono::milliseconds(2)); - lk.lock(); - search = m_catalogNameToTag.find(lower); - if (search != m_catalogNameToTag.end()) - break; - } - if (search == m_catalogNameToTag.end()) - return nullptr; - } - else - return nullptr; - } - return &search->second; + return static_cast(*m_cookSpec).getResourceIdByName(name); } FourCC ProjectResourceFactoryBase::GetResourceTypeById(CAssetId id) const { - if (!id.IsValid()) - return {}; - - std::unique_lock lk(const_cast(this)->m_backgroundIndexMutex); - SObjectTag searchTag = {FourCC(), id}; - auto search = m_tagToPath.find(searchTag); - if (search == m_tagToPath.end()) - { - if (m_backgroundRunning) - { - while (m_backgroundRunning) - { - lk.unlock(); - std::this_thread::sleep_for(std::chrono::milliseconds(2)); - lk.lock(); - search = m_tagToPath.find(searchTag); - if (search != m_tagToPath.end()) - break; - } - if (search == m_tagToPath.end()) - return {}; - } - else - return {}; - } - - return search->first.type; + return static_cast(*m_cookSpec).getResourceTypeById(id); } void ProjectResourceFactoryBase::EnumerateResources(const std::function& lambda) const { - std::unique_lock lk(const_cast(this)->m_backgroundIndexMutex); - while (m_backgroundRunning) - { - lk.unlock(); - std::this_thread::sleep_for(std::chrono::milliseconds(2)); - lk.lock(); - } - for (const auto& pair : m_tagToPath) - { - if (!lambda(pair.first)) - break; - } + return static_cast(*m_cookSpec).enumerateResources(lambda); } void ProjectResourceFactoryBase::EnumerateNamedResources( const std::function& lambda) const { - std::unique_lock lk(const_cast(this)->m_backgroundIndexMutex); - while (m_backgroundRunning) - { - lk.unlock(); - std::this_thread::sleep_for(std::chrono::milliseconds(2)); - lk.lock(); - } - lk.unlock(); - for (const auto& pair : m_catalogNameToTag) - { - if (!lambda(pair.first, pair.second)) - break; - } + return static_cast(*m_cookSpec).enumerateNamedResources(lambda); } template bool ProjectResourceFactoryBase::AsyncPumpTask(ItType& it) { /* Ensure requested resource is in the index */ - std::unique_lock lk(m_backgroundIndexMutex); AsyncTask& task = _GetAsyncTask(it); - auto search = m_tagToPath.find(task.x0_tag); - if (search == m_tagToPath.end()) + hecl::ProjectPath path = static_cast(*m_cookSpec).pathFromTag(task.x0_tag); + if (!path) { - if (!m_backgroundRunning) + if (!static_cast(*m_cookSpec).backgroundIndexRunning()) { Log.report(logvisor::Error, _S("unable to find async load resource (%s, %08X)"), task.x0_tag.type.toString().c_str(), task.x0_tag.id); @@ -1031,8 +504,7 @@ bool ProjectResourceFactoryBase::AsyncPumpTask(ItType& it) } return true; } - lk.unlock(); - task.EnsurePath(task.x0_tag, search->second); + task.EnsurePath(task.x0_tag, path); /* Pump load pipeline (cooking if needed) */ if (task.AsyncPump()) diff --git a/Editor/ProjectResourceFactoryBase.hpp b/Editor/ProjectResourceFactoryBase.hpp index 4a2a24ac5..4ee75ab56 100644 --- a/Editor/ProjectResourceFactoryBase.hpp +++ b/Editor/ProjectResourceFactoryBase.hpp @@ -6,6 +6,7 @@ #include "Runtime/IFactory.hpp" #include "Runtime/CFactoryMgr.hpp" #include "Runtime/CResFactory.hpp" +#include "DataSpec/SpecBase.hpp" #include "optional.hpp" #include @@ -72,11 +73,6 @@ public: }; protected: - std::unordered_map m_tagToPath; - std::unordered_map m_pathToTag; - std::unordered_map m_catalogNameToTag; - void Clear(); - const hecl::Database::Project* m_proj = nullptr; const hecl::Database::DataSpecEntry* m_origSpec = nullptr; const hecl::Database::DataSpecEntry* m_pcSpec = nullptr; @@ -84,11 +80,6 @@ protected: std::unique_ptr m_cookSpec; urde::CFactoryMgr m_factoryMgr; - hecl::BlenderToken m_backgroundBlender; - std::thread m_backgroundIndexTh; - std::mutex m_backgroundIndexMutex; - bool m_backgroundRunning = false; - std::list> m_asyncLoadList; std::unordered_map>::iterator> m_asyncLoadMap; std::shared_ptr @@ -111,36 +102,36 @@ protected: return **it->second; } - bool WaitForTagReady(const urde::SObjectTag& tag, const hecl::ProjectPath*& pathOut); bool PrepForReadSync(const SObjectTag& tag, const hecl::ProjectPath& path, std::experimental::optional& fr); - SObjectTag TagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const; + bool WaitForTagReady(const urde::SObjectTag& tag, const hecl::ProjectPath*& pathOut) + { + return static_cast(*m_cookSpec).waitForTagReady(tag, pathOut); + } + SObjectTag TagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const + { + return static_cast(*m_cookSpec).tagFromPath(path, btok); + } SObjectTag BuildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const { - return static_cast(*m_cookSpec).BuildTagFromPath(path, btok); + return static_cast(*m_cookSpec).buildTagFromPath(path, btok); + } + void GetTagListForFile(const char* pakName, std::vector& out) const + { + return static_cast(*m_cookSpec).getTagListForFile(pakName, out); + } + void CancelBackgroundIndex() + { + if (m_cookSpec) + return static_cast(*m_cookSpec).cancelBackgroundIndex(); } - - void ReadCatalog(const hecl::ProjectPath& catalogPath, - athena::io::YAMLDocWriter& nameWriter); - bool AddFileToIndex(const hecl::ProjectPath& path, - athena::io::YAMLDocWriter& cacheWriter); - void BackgroundIndexRecursiveProc(const hecl::ProjectPath& path, - athena::io::YAMLDocWriter& cacheWriter, - athena::io::YAMLDocWriter& nameWriter, - int level); - void BackgroundIndexRecursiveCatalogs(const hecl::ProjectPath& path, - athena::io::YAMLDocWriter& nameWriter, - int level); - void BackgroundIndexProc(); - void CancelBackgroundIndex(); void BeginBackgroundIndex(hecl::Database::Project& proj, - const hecl::Database::DataSpecEntry& origSpec, - const hecl::Database::DataSpecEntry& pcSpec); + const hecl::Database::DataSpecEntry& origSpec, + const hecl::Database::DataSpecEntry& pcSpec); - hecl::ProjectPath GetCookedPath(const hecl::ProjectPath& working, bool pcTarget) const; bool SyncCook(const hecl::ProjectPath& working); CFactoryFnReturn BuildSync(const SObjectTag& tag, const hecl::ProjectPath& path, const CVParamTransfer& paramXfer, CObjectReference* selfRef); @@ -154,6 +145,10 @@ public: bool CanBuild(const urde::SObjectTag&); const urde::SObjectTag* GetResourceIdByName(const char*) const; FourCC GetResourceTypeById(CAssetId id) const; + hecl::ProjectPath GetCookedPath(const hecl::ProjectPath& working, bool pcTarget) const + { + return static_cast(*m_cookSpec).getCookedPath(working, pcTarget); + } void EnumerateResources(const std::function& lambda) const; void EnumerateNamedResources(const std::function& lambda) const; diff --git a/Editor/ProjectResourceFactoryMP1.cpp b/Editor/ProjectResourceFactoryMP1.cpp index 317660304..82976f9df 100644 --- a/Editor/ProjectResourceFactoryMP1.cpp +++ b/Editor/ProjectResourceFactoryMP1.cpp @@ -135,19 +135,6 @@ void ProjectResourceFactoryMP1::IndexMP1Resources(hecl::Database::Project& proj, m_origIds = sp.GetObj("MP1OriginalIDs"); } -void ProjectResourceFactoryMP1::GetTagListForFile(const char* pakName, std::vector& out) const -{ - std::string pathPrefix("MP1/"); - pathPrefix += pakName; - pathPrefix += '/'; - - std::unique_lock lk( - const_cast(*this).m_backgroundIndexMutex); - for (const auto& tag : m_tagToPath) - if (!tag.second.getRelativePathUTF8().compare(0, pathPrefix.size(), pathPrefix)) - out.push_back(tag.first); -} - void ProjectResourceFactoryMP1::Shutdown() { m_origIds = TLockedToken(); diff --git a/Editor/ProjectResourceFactoryMP1.hpp b/Editor/ProjectResourceFactoryMP1.hpp index 80e87384d..bcb4d36c5 100644 --- a/Editor/ProjectResourceFactoryMP1.hpp +++ b/Editor/ProjectResourceFactoryMP1.hpp @@ -15,7 +15,6 @@ class ProjectResourceFactoryMP1 : public ProjectResourceFactoryBase public: ProjectResourceFactoryMP1(hecl::ClientProcess& clientProc); void IndexMP1Resources(hecl::Database::Project& proj, CSimplePool& sp); - void GetTagListForFile(const char* pakName, std::vector& out) const; void Shutdown(); CAssetId TranslateOriginalToNew(CAssetId id) const; diff --git a/Runtime/CDvdFile.hpp b/Runtime/CDvdFile.hpp index 5438bc10c..1be5ef174 100644 --- a/Runtime/CDvdFile.hpp +++ b/Runtime/CDvdFile.hpp @@ -2,6 +2,7 @@ #define __URDE_CDVDFILE_HPP__ #include "RetroTypes.hpp" +#include "athena/FileReader.hpp" #include #include diff --git a/Runtime/CSortedLists.hpp b/Runtime/CSortedLists.hpp index 8d151daa1..3f487f93d 100644 --- a/Runtime/CSortedLists.hpp +++ b/Runtime/CSortedLists.hpp @@ -20,7 +20,7 @@ enum ESortedList struct SSortedList { s16 x0_ids[1024]; - u32 x800_size; + u32 x800_size = 0; void Reset() {std::fill(std::begin(x0_ids), std::end(x0_ids), -1);} SSortedList() {Reset();} }; diff --git a/Runtime/CStaticInterference.cpp b/Runtime/CStaticInterference.cpp index 765c3c91d..1a65c5b63 100644 --- a/Runtime/CStaticInterference.cpp +++ b/Runtime/CStaticInterference.cpp @@ -1,4 +1,5 @@ #include "CStaticInterference.hpp" +#include "zeus/Math.hpp" namespace urde { @@ -24,7 +25,7 @@ void CStaticInterference::Update(CStateManager&, float dt) newSources.reserve(m_sources.size()); for (CStaticInterferenceSource& src : m_sources) { - if (src.timeLeft >= 0.0) + if (src.timeLeft >= 0.f) { src.timeLeft -= dt; newSources.push_back(src); @@ -35,8 +36,8 @@ void CStaticInterference::Update(CStateManager&, float dt) float CStaticInterference::GetTotalInterference() const { - float validAccum = 0.0; - float invalidAccum = 0.0; + float validAccum = 0.f; + float invalidAccum = 0.f; for (const CStaticInterferenceSource& src : m_sources) { if (src.id == kInvalidUniqueId) @@ -44,11 +45,11 @@ float CStaticInterference::GetTotalInterference() const else validAccum += src.magnitude; } - if (validAccum > 0.80000001) - validAccum = 0.80000001; + if (validAccum > 0.80000001f) + validAccum = 0.80000001f; validAccum += invalidAccum; - if (validAccum > 1.0) - return 1.0; + if (validAccum > 1.f) + return 1.f; return validAccum; } diff --git a/Runtime/Camera/CCameraSpline.hpp b/Runtime/Camera/CCameraSpline.hpp index 64399fedf..b0783d9f6 100644 --- a/Runtime/Camera/CCameraSpline.hpp +++ b/Runtime/Camera/CCameraSpline.hpp @@ -2,6 +2,7 @@ #define __URDE_CCAMERASPLINE_HPP__ #include "World/CEntityInfo.hpp" +#include "zeus/CVector3f.hpp" namespace urde { diff --git a/Runtime/Character/CBodyStateCmdMgr.cpp b/Runtime/Character/CBodyStateCmdMgr.cpp index f7c163648..3ab90d1dd 100644 --- a/Runtime/Character/CBodyStateCmdMgr.cpp +++ b/Runtime/Character/CBodyStateCmdMgr.cpp @@ -1,4 +1,5 @@ #include "CBodyStateCmdMgr.hpp" +#include namespace urde { diff --git a/Runtime/GuiSys/CHudBossEnergyInterface.hpp b/Runtime/GuiSys/CHudBossEnergyInterface.hpp index 503f6dff1..844c0f780 100644 --- a/Runtime/GuiSys/CHudBossEnergyInterface.hpp +++ b/Runtime/GuiSys/CHudBossEnergyInterface.hpp @@ -2,6 +2,7 @@ #define __URDE_CHUDBOSSENERGYINTERFACE_HPP__ #include "RetroTypes.hpp" +#include "zeus/CVector3f.hpp" namespace urde { diff --git a/Runtime/GuiSys/CHudEnergyInterface.hpp b/Runtime/GuiSys/CHudEnergyInterface.hpp index 7e26c3aa5..246afda4d 100644 --- a/Runtime/GuiSys/CHudEnergyInterface.hpp +++ b/Runtime/GuiSys/CHudEnergyInterface.hpp @@ -3,6 +3,7 @@ #include "RetroTypes.hpp" #include "CHudInterface.hpp" +#include "zeus/CVector3f.hpp" namespace urde { diff --git a/Runtime/GuiSys/CHudVisorBeamMenu.hpp b/Runtime/GuiSys/CHudVisorBeamMenu.hpp index 77d01a940..a86eed863 100644 --- a/Runtime/GuiSys/CHudVisorBeamMenu.hpp +++ b/Runtime/GuiSys/CHudVisorBeamMenu.hpp @@ -2,6 +2,7 @@ #define __URDE_CHUDVISORBEAMMENU_HPP__ #include "RetroTypes.hpp" +#include namespace urde { diff --git a/Runtime/MP1/CNESEmulator.hpp b/Runtime/MP1/CNESEmulator.hpp index fc820702a..837a39949 100644 --- a/Runtime/MP1/CNESEmulator.hpp +++ b/Runtime/MP1/CNESEmulator.hpp @@ -2,6 +2,7 @@ #define __URDE_CNESEMULATOR_HPP__ #include "RetroTypes.hpp" +#include "zeus/CColor.hpp" namespace urde { diff --git a/Runtime/RetroTypes.hpp b/Runtime/RetroTypes.hpp index beb9a6112..d70eb3f00 100644 --- a/Runtime/RetroTypes.hpp +++ b/Runtime/RetroTypes.hpp @@ -4,10 +4,11 @@ #include #include #include +#include #include "GCNTypes.hpp" #include "rstl.hpp" -#include "DataSpec/DNACommon/DNACommon.hpp" #include "IOStreams.hpp" +#include "hecl/hecl.hpp" namespace urde { @@ -24,10 +25,9 @@ public: explicit CAssetId(CInputStream& in); bool IsValid() const { return id != UINT64_MAX; } u64 Value() const { return id; } - void Assign(u64 v) { id = (v == UINT32_MAX ? UINT64_MAX : (v == 0 ? UINT64_MAX : v)); } + void Assign(u64 v) { id = (v == UINT32_MAX ? UINT64_MAX : (v == 0 ? UINT64_MAX : v)); } void Reset() { id = UINT64_MAX; } void PutTo(COutputStream& out); - bool operator==(const CAssetId& other) const { return id == other.id; } bool operator!=(const CAssetId& other) const { return id != other.id; } bool operator<(const CAssetId& other) const { return id < other.id; } @@ -39,6 +39,7 @@ struct SObjectTag { FourCC type; CAssetId id; + operator bool() const { return id.IsValid(); } bool operator!=(const SObjectTag& other) const { return id != other.id; } bool operator==(const SObjectTag& other) const { return id == other.id; } @@ -65,11 +66,11 @@ struct TEditorId u8 LayerNum() const { return u8((id >> 26) & 0x3f); } u16 AreaNum() const { return u16((id >> 16) & 0x3ff); } u16 Id() const { return u16(id & 0xffff); } - bool operator<(const TEditorId& other) const { return (id & 0x3ffffff) < (other.id & 0x3ffffff); } bool operator!=(const TEditorId& other) const { return (id & 0x3ffffff) != (other.id & 0x3ffffff); } bool operator==(const TEditorId& other) const { return (id & 0x3ffffff) == (other.id & 0x3ffffff); } }; + #define kInvalidEditorId TEditorId() struct TUniqueId @@ -77,9 +78,8 @@ struct TUniqueId TUniqueId() = default; TUniqueId(u16 value, u16 version) : id(value | (version << 10)) {} u16 id = u16(-1); - - u16 Version() const { return u16((id >> 10) & 0x3f);} - u16 Value() const { return u16(id & 0x3ff);} + u16 Version() const { return u16((id >> 10) & 0x3f); } + u16 Value() const { return u16(id & 0x3ff); } bool operator<(const TUniqueId& other) const { return (id < other.id); } bool operator!=(const TUniqueId& other) const { return (id != other.id); } bool operator==(const TUniqueId& other) const { return (id == other.id); } @@ -90,7 +90,6 @@ struct TUniqueId using TAreaId = s32; #define kInvalidAreaId TAreaId(-1) -} #if 0 template @@ -113,7 +112,7 @@ public: }; #endif -template +template T GetAverage(const T* v, s32 count) { T r = v[0]; @@ -123,11 +122,12 @@ T GetAverage(const T* v, s32 count) return r / count; } -template +template class TReservedAverage : rstl::reserved_vector { public: TReservedAverage() = default; + TReservedAverage(const T& t) { rstl::reserved_vector::resize(N, t); } void AddValue(const T& t) @@ -135,8 +135,7 @@ public: if (this->size() < N) { this->insert(this->begin(), t); - } - else + } else { this->pop_back(); this->insert(this->begin(), t); @@ -148,7 +147,7 @@ public: if (this->empty()) return {}; - return {::GetAverage(this->data(), this->size())}; + return {urde::GetAverage(this->data(), this->size())}; } rstl::optional_object GetEntry(int i) const @@ -163,18 +162,20 @@ public: size_t Size() const { return this->size(); } }; +} + namespace std { template <> struct hash { - inline size_t operator()(const urde::SObjectTag& tag) const { return tag.id.Value(); } + size_t operator()(const urde::SObjectTag& tag) const noexcept { return tag.id.Value(); } }; template <> struct hash { - inline size_t operator()(const urde::CAssetId& id) const { return id.Value(); } + size_t operator()(const urde::CAssetId& id) const noexcept { return id.Value(); } }; } diff --git a/Runtime/World/CAiFuncMap.hpp b/Runtime/World/CAiFuncMap.hpp index a05c94d8d..3eccf66c2 100644 --- a/Runtime/World/CAiFuncMap.hpp +++ b/Runtime/World/CAiFuncMap.hpp @@ -2,6 +2,7 @@ #define __URDE_CAIFUNCMAP_HPP__ #include "RetroTypes.hpp" +#include namespace urde { diff --git a/Runtime/World/CVisorFlare.hpp b/Runtime/World/CVisorFlare.hpp index 995d495a2..42737e882 100644 --- a/Runtime/World/CVisorFlare.hpp +++ b/Runtime/World/CVisorFlare.hpp @@ -3,6 +3,7 @@ #include "RetroTypes.hpp" #include "CToken.hpp" +#include "zeus/CColor.hpp" namespace urde { diff --git a/Runtime/World/CWorldLight.cpp b/Runtime/World/CWorldLight.cpp index 117ea1b38..78eb32703 100644 --- a/Runtime/World/CWorldLight.cpp +++ b/Runtime/World/CWorldLight.cpp @@ -1,4 +1,5 @@ #include "CWorldLight.hpp" +#include namespace urde { diff --git a/hecl b/hecl index 23a08dcf4..70b73855b 160000 --- a/hecl +++ b/hecl @@ -1 +1 @@ -Subproject commit 23a08dcf40456c59ff40710d12241a8a8735fe5e +Subproject commit 70b73855bf906c81d18bdf70c9b701bba996314e