AxioDL
/
metaforce
mirror of https://github.com/AxioDL/metaforce.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

Refactorings to support .upak generation

This commit is contained in:
Jack Andersen 2017-10-24 21:47:49 -10:00
parent 4ee4963aaf
commit e274cd12b9
28 changed files with 1174 additions and 644 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
DataSpec/DNAMP1/ANCS.cpp
View File

@ -1130,7 +1130,8 @@ bool ANCS::Cook(const hecl::ProjectPath& outPath,
if (sub.overlayMeshes.size())
{
ch.cmdlOverlay = sub.overlayMeshes[0].second;
ch.cskrOverlay = inPath.ensureAuxInfo(chSysName.sys_str() + _S(".over.CSKR"));
ch.cskrOverlay = inPath.ensureAuxInfo(chSysName.sys_str() + _S('.') +
sub.overlayMeshes[0].first + _S(".CSKR"));
}
break;
@ -1206,7 +1207,15 @@ bool ANCS::CookCSKR(const hecl::ProjectPath& outPath,
{
hecl::SystemString subName(inPath.getAuxInfo().begin(),
inPath.getAuxInfo().end() - 5);
hecl::SystemString overName;
auto dotPos = subName.rfind(_S('.'));
if (dotPos != hecl::SystemString::npos)
{
overName = hecl::SystemString(subName.begin() + dotPos + 1, subName.end());
subName = hecl::SystemString(subName.begin(), subName.begin() + dotPos);
}
hecl::SystemUTF8View subNameView(subName);
hecl::SystemUTF8View overNameView(overName);
/* Build bone ID map */
std::unordered_map<std::string, atInt32> boneIdMap;
@ -1227,15 +1236,30 @@ bool ANCS::CookCSKR(const hecl::ProjectPath& outPath,
if (!subtype)
Log.report(logvisor::Fatal, _S("unable to find subtype '%s'"), subName.c_str());
const hecl::ProjectPath& modelPath = subtype->mesh;
const hecl::ProjectPath* modelPath = nullptr;
if (overName.empty())
{
modelPath = &subtype->mesh;
}
else
{
for (const auto& overlay : subtype->overlayMeshes)
if (!overlay.first.compare(overNameView.str()))
{
modelPath = &overlay.second;
break;
}
}
if (!modelPath)
Log.report(logvisor::Fatal, _S("unable to resolve model path of %s:%s"), subName.c_str(), overName.c_str());
if (!modelPath.isFile())
Log.report(logvisor::Fatal, _S("unable to resolve '%s'"), modelPath.getRelativePath().c_str());
if (!modelPath->isFile())
Log.report(logvisor::Fatal, _S("unable to resolve '%s'"), modelPath->getRelativePath().c_str());
hecl::ProjectPath skinIntPath = modelPath.getCookedPath(SpecEntMP1PC).getWithExtension(_S(".skinint"));
if (!skinIntPath.isFileOrGlob() || skinIntPath.getModtime() < modelPath.getModtime())
if (!modelCookFunc(modelPath))
Log.report(logvisor::Fatal, _S("unable to cook '%s'"), modelPath.getRelativePath().c_str());
hecl::ProjectPath skinIntPath = modelPath->getCookedPath(SpecEntMP1PC).getWithExtension(_S(".skinint"));
if (!skinIntPath.isFileOrGlob() || skinIntPath.getModtime() < modelPath->getModtime())
if (!modelCookFunc(*modelPath))
Log.report(logvisor::Fatal, _S("unable to cook '%s'"), modelPath->getRelativePath().c_str());
athena::io::FileReader skinIO(skinIntPath.getAbsolutePath(), 1024*32, false);
if (skinIO.hasError())

2
DataSpec/DNAMP1/DNAMP1.cpp
View File

@ -244,7 +244,7 @@ void PAKBridge::addCMDLRigPairs(PAKRouter<PAKBridge>& pakRouter,
if (ci.cmdlOverlay && ci.cskrOverlay)
{
addTo[ci.cmdlOverlay] = std::make_pair(ci.cskrOverlay, ci.cinf);
cskrCinfToAncs[ci.cskrOverlay] = std::make_pair(entry.second.id, hecl::Format("%s.over.CSKR", ci.name.c_str()));
cskrCinfToAncs[ci.cskrOverlay] = std::make_pair(entry.second.id, hecl::Format("%s.OVER.CSKR", ci.name.c_str()));
PAK::Entry* cmdlEnt = (PAK::Entry*)m_pak.lookupEntry(ci.cmdlOverlay);
PAK::Entry* cskrEnt = (PAK::Entry*)m_pak.lookupEntry(ci.cskrOverlay);
cmdlEnt->name = hecl::Format("ANCS_%08X_%s_overmodel", entry.first.toUint32(), ci.name.c_str());

13
DataSpec/DNAMP1/MLVL.cpp
View File

@ -73,9 +73,10 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
hecl::ProjectPath namePath(inPath.getParentPath(), _S("!name.yaml"));
if (namePath.isFile())
mlvl.worldNameId = namePath;
hecl::ProjectPath savwPath = inPath.ensureAuxInfo(_S("SAVW"));
hecl::ProjectPath globPath = inPath.getWithExtension(_S(".*"), true);
hecl::ProjectPath savwPath = globPath.ensureAuxInfo(_S("SAVW"));
mlvl.saveWorldId = savwPath;
hecl::ProjectPath mapwPath = inPath.ensureAuxInfo(_S("MAPW"));
hecl::ProjectPath mapwPath = globPath.ensureAuxInfo(_S("MAPW"));
mlvl.worldMap = mapwPath;
size_t areaIdx = 0;
@ -269,7 +270,7 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
if (addedPaths.find(path.hash()) == addedPaths.cend())
{
addedPaths.insert(path.hash());
urde::SObjectTag tag = g_curSpec->BuildTagFromPath(path, btok);
urde::SObjectTag tag = g_curSpec->buildTagFromPath(path, btok);
if (tag.id.IsValid())
areaOut.deps.emplace_back(tag.id.Value(), tag.type);
}
@ -308,13 +309,13 @@ bool MLVL::Cook(const hecl::ProjectPath& outPath, const hecl::ProjectPath& inPat
if (addedPaths.find(path.hash()) == addedPaths.cend())
{
addedPaths.insert(path.hash());
urde::SObjectTag tag = g_curSpec->BuildTagFromPath(path, btok);
urde::SObjectTag tag = g_curSpec->buildTagFromPath(path, btok);
if (tag.id.IsValid())
areaOut.deps.emplace_back(tag.id.Value(), tag.type);
}
}
urde::SObjectTag tag = g_curSpec->BuildTagFromPath(areaPath, btok);
urde::SObjectTag tag = g_curSpec->buildTagFromPath(areaPath, btok);
if (tag.id.IsValid())
areaOut.deps.emplace_back(tag.id.Value(), tag.type);
}
@ -359,7 +360,7 @@ bool MLVL::CookMAPW(const hecl::ProjectPath& outPath,
/* Area map */
hecl::ProjectPath mapPath(area.path, _S("/!map.blend"));
if (mapPath.isFile())
mapaTags.push_back(g_curSpec->BuildTagFromPath(mapPath, btok));
mapaTags.push_back(g_curSpec->buildTagFromPath(mapPath, btok));
}
/* Write out MAPW */

4
DataSpec/DNAMP1/MREA.cpp
View File

@ -562,9 +562,9 @@ bool MREA::PCCook(const hecl::ProjectPath& outPath,
hecl::SNPrintf(thrIdx, 16, _S("%d"), hecl::ClientProcess::GetThreadWorkerIdx());
hecl::SystemChar parPid[32];
#if _WIN32
hecl::SNPrintf(parPid, 32, _S("%ullX"), reinterpret_cast<unsigned long long>(GetCurrentProcess()));
hecl::SNPrintf(parPid, 32, _S("%lluX"), reinterpret_cast<unsigned long long>(GetCurrentProcess()));
#else
hecl::SNPrintf(parPid, 32, _S("%ullX"), reinterpret_cast<unsigned long long>(getpid()));
hecl::SNPrintf(parPid, 32, _S("%lluX"), (unsigned long long)getpid());
#endif
const hecl::SystemChar* args[] = {VisiGenPath.c_str(),
visiIntOut.getAbsolutePath().c_str(),

2
DataSpec/DNAMP2/DNAMP2.cpp
View File

@ -191,7 +191,7 @@ void PAKBridge::addCMDLRigPairs(PAKRouter<PAKBridge>& pakRouter,
if (ci.cmdlOverlay)
{
addTo[ci.cmdlOverlay] = std::make_pair(ci.cskrOverlay, ci.cinf);
cskrCinfToAncs[ci.cskrOverlay] = std::make_pair(entry.second.id, hecl::Format("%s.over.CSKR", ci.name.c_str()));
cskrCinfToAncs[ci.cskrOverlay] = std::make_pair(entry.second.id, hecl::Format("%s.OVER.CSKR", ci.name.c_str()));
}
}
}

770
DataSpec/SpecBase.cpp
View File

@ -5,14 +5,15 @@
#include "SpecBase.hpp"
#include "Blender/BlenderSupport.hpp"
#include "hecl/Blender/BlenderConnection.hpp"
#include "DNACommon/DNACommon.hpp"
#include "DNACommon/TXTR.hpp"
#include "AssetNameMap.hpp"
#include "hecl/ClientProcess.hpp"
#include <time.h>
#include <png.h>
#define DUMP_CACHE_FILL 1
namespace DataSpec
{
@ -43,6 +44,11 @@ SpecBase::SpecBase(const hecl::Database::DataSpecEntry* specEntry, hecl::Databas
DataSpec::UniqueIDBridge::setThreadProject(m_project);
}
SpecBase::~SpecBase()
{
cancelBackgroundIndex();
}
static const hecl::SystemString regNONE = _S("");
static const hecl::SystemString regE = _S("NTSC");
static const hecl::SystemString regJ = _S("NTSC-J");
@ -133,7 +139,15 @@ static bool IsPathAudioGroup(const hecl::ProjectPath& path)
!path.getWithExtension(_S(".proj"), true).isFile() ||
!path.getWithExtension(_S(".sdir"), true).isFile() ||
!path.getWithExtension(_S(".samp"), true).isFile())
{
if (path.isFile() &&
!hecl::StrCmp(_S("proj"), path.getLastComponentExt()) &&
path.getWithExtension(_S(".pool"), true).isFile() &&
path.getWithExtension(_S(".sdir"), true).isFile() &&
path.getWithExtension(_S(".samp"), true).isFile())
return true;
return false;
}
return true;
}
@ -142,7 +156,13 @@ static bool IsPathSong(const hecl::ProjectPath& path)
if (path.getPathType() != hecl::ProjectPath::Type::Glob ||
!path.getWithExtension(_S(".mid"), true).isFile() ||
!path.getWithExtension(_S(".yaml"), true).isFile())
{
if (path.isFile() &&
!hecl::StrCmp(_S("mid"), path.getLastComponentExt()) &&
path.getWithExtension(_S(".yaml"), true).isFile())
return true;
return false;
}
return true;
}
@ -359,10 +379,11 @@ void SpecBase::flattenDependencies(const hecl::ProjectPath& path,
const auto& arm = actor.armatures[sub.armature];
hecl::SystemStringView armSysName(arm.name);
pathsOut.push_back(path.ensureAuxInfo(armSysName.sys_str() + _S(".CINF")));
if (sub.overlayMeshes.size())
for (const auto& overlay : sub.overlayMeshes)
{
pathsOut.push_back(sub.overlayMeshes[0].second);
pathsOut.push_back(path.ensureAuxInfo(chSysName.sys_str() + _S(".over.CSKR")));
pathsOut.push_back(overlay.second);
pathsOut.push_back(path.ensureAuxInfo(chSysName.sys_str() + _S('.') +
overlay.first + _S(".CSKR")));
}
}
}
@ -402,13 +423,180 @@ void SpecBase::flattenDependencies(const UniqueID64& id, std::vector<hecl::Proje
flattenDependencies(path, pathsOut, *g_ThreadBlenderToken.get());
}
bool SpecBase::canPackage(const PackagePassInfo& info)
bool SpecBase::canPackage(const hecl::ProjectPath& path)
{
auto components = path.getPathComponents();
if (components.size() <= 1)
return false;
hecl::ProjectPath outDir(m_project.getProjectWorkingPath(), _S("out"));
if (path.getPathType() == hecl::ProjectPath::Type::File &&
!hecl::StrCmp(path.getLastComponent(), _S("!world.blend")))
return true;
else if (path.getPathType() == hecl::ProjectPath::Type::Directory)
return true;
return false;
}
void SpecBase::doPackage(const PackagePassInfo& info)
void SpecBase::recursiveBuildResourceList(std::vector<urde::SObjectTag>& listOut,
const hecl::ProjectPath& path,
hecl::BlenderToken& btok)
{
hecl::DirectoryEnumerator dEnum(path.getAbsolutePath(),
hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted, false, false, true);
for (const auto& ent : dEnum)
{
hecl::ProjectPath childPath(path, ent.m_name);
if (ent.m_isDir)
recursiveBuildResourceList(listOut, childPath, btok);
else if (urde::SObjectTag tag = tagFromPath(childPath, btok))
listOut.push_back(tag);
}
}
void SpecBase::copyBuildListData(std::vector<std::tuple<size_t, size_t, bool>>& fileIndex,
const std::vector<urde::SObjectTag>& buildList,
const hecl::Database::DataSpecEntry* entry,
bool fast, FProgress progress, athena::io::FileWriter& pakOut)
{
fileIndex.reserve(buildList.size());
size_t loadIdx = 0;
for (const auto& tag : buildList)
{
fprintf(stderr, "\r %" PRISize " / %" PRISize " %.4s %08X", ++loadIdx, buildList.size(),
tag.type.getChars(), (unsigned int)tag.id.Value());
fileIndex.emplace_back();
auto& thisIdx = fileIndex.back();
hecl::ProjectPath path = pathFromTag(tag);
hecl::ProjectPath cooked = getCookedPath(path, true);
athena::io::FileReader r(cooked.getAbsolutePath());
if (r.hasError())
Log.report(logvisor::Fatal, _S("Unable to open resource %s"), cooked.getRelativePath().c_str());
atUint64 size = r.length();
auto data = r.readUBytes(size);
auto compData = compressPakData(tag, data.get(), size);
if (compData.first)
{
std::get<0>(thisIdx) = pakOut.position();
std::get<1>(thisIdx) = ROUND_UP_32(compData.second + 4);
std::get<2>(thisIdx) = true;
pakOut.writeUint32Big(atUint32(size));
pakOut.writeUBytes(compData.first.get(), compData.second);
for (atUint64 i = compData.second + 4 ; i < std::get<1>(thisIdx) ; ++i)
pakOut.writeUByte(0xff);
}
else
{
std::get<0>(thisIdx) = pakOut.position();
std::get<1>(thisIdx) = ROUND_UP_32(size);
std::get<2>(thisIdx) = false;
pakOut.writeUBytes(data.get(), size);
for (atUint64 i = size ; i < std::get<1>(thisIdx) ; ++i)
pakOut.writeUByte(0xff);
}
}
fprintf(stderr, "\n");
}
void SpecBase::doPackage(const hecl::ProjectPath& path, const hecl::Database::DataSpecEntry* entry,
bool fast, hecl::BlenderToken& btok, FProgress progress, hecl::ClientProcess* cp)
{
/* Prepare complete resource index */
if (!m_backgroundRunning && m_tagToPath.empty())
beginBackgroundIndex();
waitForIndexComplete();
/* Name pak based on root-relative components */
auto components = path.getPathComponents();
if (components.size() <= 1)
return;
hecl::ProjectPath outPath(m_project.getProjectWorkingPath(),
_S("out/") + components[0] + _S("/") + components[1] + _S(".upak"));
outPath.makeDirChain(false);
hecl::SystemString tmpPath = outPath.getAbsolutePath() + _S("~");
/* Output file */
athena::io::FileWriter pakOut(tmpPath);
std::vector<urde::SObjectTag> buildList;
atUint64 resTableOffset = 0;
if (path.getPathType() == hecl::ProjectPath::Type::File &&
!hecl::StrCmp(path.getLastComponent(), _S("!world.blend"))) /* World PAK */
{
/* Force-cook MLVL and write resource list structure */
m_project.cookPath(path, progress, false, true, fast);
hecl::ProjectPath cooked = getCookedPath(path, true);
buildWorldPakList(path, cooked, btok, pakOut, buildList, resTableOffset);
if (int64_t rem = pakOut.position() % 32)
for (int64_t i=0 ; i<32-rem ; ++i)
pakOut.writeUByte(0xff);
}
else if (path.getPathType() == hecl::ProjectPath::Type::Directory) /* General PAK */
{
/* Build resource list */
recursiveBuildResourceList(buildList, path, btok);
std::vector<std::pair<urde::SObjectTag, std::string>> nameList;
/* Build name list */
for (const auto& item : buildList)
{
auto search = m_catalogTagToName.find(item);
if (search != m_catalogTagToName.end())
nameList.emplace_back(item, search->second);
}
/* Write resource list structure */
buildPakList(btok, pakOut, buildList, nameList, resTableOffset);
if (int64_t rem = pakOut.position() % 32)
for (int64_t i=0 ; i<32-rem ; ++i)
pakOut.writeUByte(0xff);
}
/* Async cook resource list if using ClientProcess */
if (cp)
{
Log.report(logvisor::Info, _S("Validating resources"));
size_t loadIdx = 0;
for (auto& tag : buildList)
{
fprintf(stderr, "\r %" PRISize " / %" PRISize " %.4s %08X", ++loadIdx, buildList.size(),
tag.type.getChars(), (unsigned int)tag.id.Value());
hecl::ProjectPath depPath = pathFromTag(tag);
if (!depPath)
{
fprintf(stderr, "\n");
Log.report(logvisor::Fatal, _S("Unable to resolve %.4s %08X"),
tag.type.getChars(), tag.id.Value());
}
m_project.cookPath(depPath, progress, false, false, fast, cp);
}
fprintf(stderr, "\n");
Log.report(logvisor::Info, _S("Waiting for remaining cook transactions"));
cp->waitUntilComplete();
}
/* Write resource data and build file index */
std::vector<std::tuple<size_t, size_t, bool>> fileIndex;
Log.report(logvisor::Info, _S("Copying data into %s"), outPath.getRelativePath().c_str());
copyBuildListData(fileIndex, buildList, entry, fast, progress, pakOut);
/* Write file index */
writePakFileIndex(pakOut, buildList, fileIndex, resTableOffset);
pakOut.close();
/* Atomic rename */
hecl::Rename(tmpPath.c_str(), outPath.getAbsolutePath().c_str());
}
hecl::ProjectPath SpecBase::getCookedPath(const hecl::ProjectPath& working, bool pcTarget) const
{
const hecl::Database::DataSpecEntry* spec = &getOriginalSpec();
if (pcTarget)
spec = overrideDataSpec(working, getDataSpecEntry(), hecl::SharedBlenderToken);
if (!spec)
return {};
return working.getCookedPath(*spec);
}
static void PNGErr(png_structp png, png_const_charp msg)
@ -427,7 +615,7 @@ static inline uint8_t Convert4To8(uint8_t v)
return (v << 4) | v;
}
void SpecBase::ExtractRandomStaticEntropy(const uint8_t* buf, const hecl::ProjectPath& noAramPath)
void SpecBase::extractRandomStaticEntropy(const uint8_t* buf, const hecl::ProjectPath& noAramPath)
{
hecl::ProjectPath entropyPath(noAramPath, _S("RandomStaticEntropy.png"));
hecl::ProjectPath catalogPath(noAramPath, _S("!catalog.yaml"));
@ -477,4 +665,570 @@ void SpecBase::ExtractRandomStaticEntropy(const uint8_t* buf, const hecl::Projec
fclose(fp);
}
void SpecBase::clearTagCache()
{
m_tagToPath.clear();
m_pathToTag.clear();
m_catalogNameToTag.clear();
m_catalogTagToName.clear();
}
hecl::ProjectPath SpecBase::pathFromTag(const urde::SObjectTag& tag) const
{
std::unique_lock<std::mutex> lk(const_cast<SpecBase&>(*this).m_backgroundIndexMutex);
auto search = m_tagToPath.find(tag);
if (search != m_tagToPath.cend())
return search->second;
return {};
}
urde::SObjectTag SpecBase::tagFromPath(const hecl::ProjectPath& path,
hecl::BlenderToken& btok) const
{
auto search = m_pathToTag.find(path.hash());
if (search != m_pathToTag.cend())
return search->second;
return buildTagFromPath(path, btok);
}
bool SpecBase::waitForTagReady(const urde::SObjectTag& tag, const hecl::ProjectPath*& pathOut)
{
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
auto search = m_tagToPath.find(tag);
if (search == m_tagToPath.end())
{
if (m_backgroundRunning)
{
while (m_backgroundRunning)
{
lk.unlock();
std::this_thread::sleep_for(std::chrono::milliseconds(2));
lk.lock();
search = m_tagToPath.find(tag);
if (search != m_tagToPath.end())
break;
}
if (search == m_tagToPath.end())
return false;
}
else
return false;
}
lk.unlock();
pathOut = &search->second;
return true;
}
const urde::SObjectTag* SpecBase::getResourceIdByName(const char* name) const
{
std::string lower = name;
std::transform(lower.cbegin(), lower.cend(), lower.begin(), tolower);
std::unique_lock<std::mutex> lk(const_cast<SpecBase&>(*this).m_backgroundIndexMutex);
auto search = m_catalogNameToTag.find(lower);
if (search == m_catalogNameToTag.end())
{
if (m_backgroundRunning)
{
while (m_backgroundRunning)
{
lk.unlock();
std::this_thread::sleep_for(std::chrono::milliseconds(2));
lk.lock();
search = m_catalogNameToTag.find(lower);
if (search != m_catalogNameToTag.end())
break;
}
if (search == m_catalogNameToTag.end())
return nullptr;
}
else
return nullptr;
}
return &search->second;
}
FourCC SpecBase::getResourceTypeById(urde::CAssetId id) const
{
if (!id.IsValid())
return {};
std::unique_lock<std::mutex> lk(const_cast<SpecBase&>(*this).m_backgroundIndexMutex);
urde::SObjectTag searchTag = {FourCC(), id};
auto search = m_tagToPath.find(searchTag);
if (search == m_tagToPath.end())
{
if (m_backgroundRunning)
{
while (m_backgroundRunning)
{
lk.unlock();
std::this_thread::sleep_for(std::chrono::milliseconds(2));
lk.lock();
search = m_tagToPath.find(searchTag);
if (search != m_tagToPath.end())
break;
}
if (search == m_tagToPath.end())
return {};
}
else
return {};
}
return search->first.type;
}
void SpecBase::enumerateResources(const std::function<bool(const urde::SObjectTag&)>& lambda) const
{
waitForIndexComplete();
for (const auto& pair : m_tagToPath)
{
if (!lambda(pair.first))
break;
}
}
void SpecBase::enumerateNamedResources(
const std::function<bool(const std::string&, const urde::SObjectTag&)>& lambda) const
{
waitForIndexComplete();
for (const auto& pair : m_catalogNameToTag)
{
if (!lambda(pair.first, pair.second))
break;
}
}
static void WriteTag(athena::io::YAMLDocWriter& cacheWriter,
const urde::SObjectTag& pathTag, const hecl::ProjectPath& path)
{
char idStr[9];
snprintf(idStr, 9, "%08X", uint32_t(pathTag.id.Value()));
if (auto v = cacheWriter.enterSubVector(idStr))
{
cacheWriter.writeString(nullptr, pathTag.type.toString().c_str());
cacheWriter.writeString(nullptr, path.getAuxInfo().size() ?
(path.getRelativePathUTF8() + '|' + path.getAuxInfoUTF8()) :
path.getRelativePathUTF8());
}
}
static void WriteNameTag(athena::io::YAMLDocWriter& nameWriter,
const urde::SObjectTag& pathTag,
const std::string& name)
{
char idStr[9];
snprintf(idStr, 9, "%08X", uint32_t(pathTag.id.Value()));
nameWriter.writeString(name.c_str(), idStr);
}
void SpecBase::readCatalog(const hecl::ProjectPath& catalogPath,
athena::io::YAMLDocWriter& nameWriter)
{
athena::io::FileReader freader(catalogPath.getAbsolutePath());
if (!freader.isOpen())
return;
athena::io::YAMLDocReader reader;
bool res = reader.parse(&freader);
if (!res)
return;
const athena::io::YAMLNode* root = reader.getRootNode();
for (const auto& p : root->m_mapChildren)
{
/* Hash as lowercase since lookup is case-insensitive */
std::string pLower = p.first;
std::transform(pLower.cbegin(), pLower.cend(), pLower.begin(), tolower);
/* Avoid redundant filesystem access for re-caches */
if (m_catalogNameToTag.find(pLower) != m_catalogNameToTag.cend())
continue;
athena::io::YAMLNode& node = *p.second;
hecl::ProjectPath path;
if (node.m_type == YAML_SCALAR_NODE)
{
path = hecl::ProjectPath(m_project.getProjectWorkingPath(), node.m_scalarString);
}
else if (node.m_type == YAML_SEQUENCE_NODE)
{
if (node.m_seqChildren.size() >= 2)
path = hecl::ProjectPath(m_project.getProjectWorkingPath(), node.m_seqChildren[0]->m_scalarString).
ensureAuxInfo(node.m_seqChildren[1]->m_scalarString);
else if (node.m_seqChildren.size() == 1)
path = hecl::ProjectPath(m_project.getProjectWorkingPath(), node.m_seqChildren[0]->m_scalarString);
}
if (!path.isFileOrGlob())
continue;
urde::SObjectTag pathTag = tagFromPath(path, m_backgroundBlender);
if (pathTag)
{
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
m_catalogNameToTag[pLower] = pathTag;
m_catalogTagToName[pathTag] = p.first;
WriteNameTag(nameWriter, pathTag, p.first);
#if 0
fprintf(stderr, "%s %s %08X\n",
p.first.c_str(),
pathTag.type.toString().c_str(), uint32_t(pathTag.id));
#endif
}
}
}
void SpecBase::backgroundIndexRecursiveCatalogs(const hecl::ProjectPath& dir,
athena::io::YAMLDocWriter& nameWriter,
int level)
{
hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(),
hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted,
false, false, true);
/* Enumerate all items */
for (const hecl::DirectoryEnumerator::Entry& ent : dEnum)
{
hecl::ProjectPath path(dir, ent.m_name);
if (ent.m_isDir && level < 1)
backgroundIndexRecursiveCatalogs(path, nameWriter, level + 1);
else
{
if (!path.isFile())
continue;
/* Read catalog.yaml for .pak directory if exists */
if (level == 1 && !ent.m_name.compare(_S("!catalog.yaml")))
{
readCatalog(path, nameWriter);
continue;
}
}
/* bail if cancelled by client */
if (!m_backgroundRunning)
break;
}
}
#if DUMP_CACHE_FILL
static void DumpCacheAdd(const urde::SObjectTag& pathTag, const hecl::ProjectPath& path)
{
fprintf(stderr, "%s %08X %s\n",
pathTag.type.toString().c_str(), uint32_t(pathTag.id.Value()),
path.getRelativePathUTF8().c_str());
}
#endif
bool SpecBase::addFileToIndex(const hecl::ProjectPath& path,
athena::io::YAMLDocWriter& cacheWriter)
{
/* Avoid redundant filesystem access for re-caches */
if (m_pathToTag.find(path.hash()) != m_pathToTag.cend())
return true;
/* Try as glob */
hecl::ProjectPath asGlob = path.getWithExtension(_S(".*"), true);
if (m_pathToTag.find(asGlob.hash()) != m_pathToTag.cend())
return true;
/* Classify intermediate into tag */
urde::SObjectTag pathTag = buildTagFromPath(path, m_backgroundBlender);
if (pathTag)
{
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
bool useGlob = false;
/* Special multi-resource intermediates */
if (pathTag.type == SBIG('ANCS'))
{
hecl::BlenderConnection& conn = m_backgroundBlender.getBlenderConnection();
if (!conn.openBlend(path) || conn.getBlendType() != hecl::BlenderConnection::BlendType::Actor)
return false;
/* Transform tag to glob */
pathTag = {SBIG('ANCS'), asGlob.hash().val32()};
useGlob = true;
hecl::BlenderConnection::DataStream ds = conn.beginData();
std::vector<std::string> armatureNames = ds.getArmatureNames();
std::vector<std::string> subtypeNames = ds.getSubtypeNames();
std::vector<std::string> actionNames = ds.getActionNames();
for (const std::string& arm : armatureNames)
{
hecl::SystemStringView sysStr(arm);
hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".CINF"));
urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender);
m_tagToPath[pathTag] = subPath;
m_pathToTag[subPath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, subPath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, subPath);
#endif
}
for (const std::string& sub : subtypeNames)
{
hecl::SystemStringView sysStr(sub);
hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".CSKR"));
urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender);
m_tagToPath[pathTag] = subPath;
m_pathToTag[subPath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, subPath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, subPath);
#endif
std::vector<std::string> overlayNames = ds.getSubtypeOverlayNames(sub);
for (const auto& overlay : overlayNames)
{
hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S('.') +
overlay + _S(".CSKR"));
urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender);
m_tagToPath[pathTag] = subPath;
m_pathToTag[subPath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, subPath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, subPath);
#endif
}
}
for (const std::string& act : actionNames)
{
hecl::SystemStringView sysStr(act);
hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".ANIM"));
urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender);
m_tagToPath[pathTag] = subPath;
m_pathToTag[subPath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, subPath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, subPath);
#endif
}
}
else if (pathTag.type == SBIG('MLVL'))
{
/* Transform tag to glob */
pathTag = {SBIG('MLVL'), asGlob.hash().val32()};
useGlob = true;
hecl::ProjectPath subPath = asGlob.ensureAuxInfo(_S("MAPW"));
urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender);
m_tagToPath[pathTag] = subPath;
m_pathToTag[subPath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, subPath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, subPath);
#endif
subPath = asGlob.ensureAuxInfo(_S("SAVW"));
pathTag = buildTagFromPath(subPath, m_backgroundBlender);
m_tagToPath[pathTag] = subPath;
m_pathToTag[subPath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, subPath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, subPath);
#endif
}
else if (pathTag.type == SBIG('AGSC'))
{
/* Transform tag to glob */
pathTag = {SBIG('AGSC'), asGlob.hash().val32()};
useGlob = true;
}
else if (pathTag.type == SBIG('MREA'))
{
hecl::ProjectPath subPath = path.ensureAuxInfo(_S("PATH"));
urde::SObjectTag pathTag = buildTagFromPath(subPath, m_backgroundBlender);
m_tagToPath[pathTag] = subPath;
m_pathToTag[subPath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, subPath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, subPath);
#endif
}
/* Cache in-memory */
const hecl::ProjectPath& usePath = useGlob ? asGlob : path;
m_tagToPath[pathTag] = usePath;
m_pathToTag[usePath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, usePath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, usePath);
#endif
}
return true;
}
void SpecBase::backgroundIndexRecursiveProc(const hecl::ProjectPath& dir,
athena::io::YAMLDocWriter& cacheWriter,
athena::io::YAMLDocWriter& nameWriter,
int level)
{
hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(),
hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted,
false, false, true);
/* Enumerate all items */
for (const hecl::DirectoryEnumerator::Entry& ent : dEnum)
{
hecl::ProjectPath path(dir, ent.m_name);
if (ent.m_isDir)
backgroundIndexRecursiveProc(path, cacheWriter, nameWriter, level + 1);
else
{
if (!path.isFile())
continue;
/* Read catalog.yaml for .pak directory if exists */
if (level == 1 && !ent.m_name.compare(_S("!catalog.yaml")))
{
readCatalog(path, nameWriter);
continue;
}
/* Index the regular file */
addFileToIndex(path, cacheWriter);
}
/* bail if cancelled by client */
if (!m_backgroundRunning)
break;
}
}
void SpecBase::backgroundIndexProc()
{
logvisor::RegisterThreadName("Resource Index Thread");
hecl::ProjectPath tagCachePath(m_project.getProjectCookedPath(getOriginalSpec()), _S("tag_cache.yaml"));
hecl::ProjectPath nameCachePath(m_project.getProjectCookedPath(getOriginalSpec()), _S("name_cache.yaml"));
hecl::ProjectPath specRoot(m_project.getProjectWorkingPath(), getOriginalSpec().m_name);
/* Cache will be overwritten with validated entries afterwards */
athena::io::YAMLDocWriter cacheWriter(nullptr);
athena::io::YAMLDocWriter nameWriter(nullptr);
/* Read in tag cache */
if (tagCachePath.isFile())
{
athena::io::FileReader reader(tagCachePath.getAbsolutePath());
if (reader.isOpen())
{
Log.report(logvisor::Info, _S("Cache index of '%s' loading"), getOriginalSpec().m_name);
athena::io::YAMLDocReader cacheReader;
if (cacheReader.parse(&reader))
{
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
size_t tagCount = cacheReader.getRootNode()->m_mapChildren.size();
m_tagToPath.reserve(tagCount);
m_pathToTag.reserve(tagCount);
size_t loadIdx = 0;
for (const auto& child : cacheReader.getRootNode()->m_mapChildren)
{
const athena::io::YAMLNode& node = *child.second;
unsigned long id = strtoul(child.first.c_str(), nullptr, 16);
hecl::FourCC type(node.m_seqChildren.at(0)->m_scalarString.c_str());
hecl::ProjectPath path(m_project.getProjectWorkingPath(),
node.m_seqChildren.at(1)->m_scalarString);
if (path.isFileOrGlob())
{
urde::SObjectTag pathTag(type, id);
m_tagToPath[pathTag] = path;
m_pathToTag[path.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, path);
}
fprintf(stderr, "\r %" PRISize " / %" PRISize, ++loadIdx, tagCount);
}
fprintf(stderr, "\n");
}
Log.report(logvisor::Info, _S("Cache index of '%s' loaded; %d tags"),
getOriginalSpec().m_name, m_tagToPath.size());
if (nameCachePath.isFile())
{
/* Read in name cache */
Log.report(logvisor::Info, _S("Name index of '%s' loading"), getOriginalSpec().m_name);
athena::io::FileReader nreader(nameCachePath.getAbsolutePath());
athena::io::YAMLDocReader nameReader;
if (nameReader.parse(&nreader))
{
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
m_catalogNameToTag.reserve(nameReader.getRootNode()->m_mapChildren.size());
m_catalogTagToName.reserve(nameReader.getRootNode()->m_mapChildren.size());
for (const auto& child : nameReader.getRootNode()->m_mapChildren)
{
unsigned long id = strtoul(child.second->m_scalarString.c_str(), nullptr, 16);
auto search = m_tagToPath.find(urde::SObjectTag(FourCC(), uint32_t(id)));
if (search != m_tagToPath.cend())
{
std::string chLower = child.first;
std::transform(chLower.cbegin(), chLower.cend(), chLower.begin(), tolower);
m_catalogNameToTag[chLower] = search->first;
m_catalogTagToName[search->first] = child.first;
WriteNameTag(nameWriter, search->first, child.first);
}
}
}
Log.report(logvisor::Info, _S("Name index of '%s' loaded; %d names"),
getOriginalSpec().m_name, m_catalogNameToTag.size());
}
}
}
/* Add special original IDs resource if exists (not name-cached to disk) */
hecl::ProjectPath oidsPath(specRoot, "!original_ids.yaml");
urde::SObjectTag oidsTag = buildTagFromPath(oidsPath, m_backgroundBlender);
if (oidsTag)
{
m_catalogNameToTag["mp1originalids"] = oidsTag;
m_catalogTagToName[oidsTag] = "MP1OriginalIDs";
}
Log.report(logvisor::Info, _S("Background index of '%s' started"), getOriginalSpec().m_name);
backgroundIndexRecursiveProc(specRoot, cacheWriter, nameWriter, 0);
tagCachePath.makeDirChain(false);
athena::io::FileWriter twriter(tagCachePath.getAbsolutePath());
cacheWriter.finish(&twriter);
athena::io::FileWriter nwriter(nameCachePath.getAbsolutePath());
nameWriter.finish(&nwriter);
m_backgroundBlender.shutdown();
Log.report(logvisor::Info, _S("Background index of '%s' complete; %d tags, %d names"),
getOriginalSpec().m_name, m_tagToPath.size(), m_catalogNameToTag.size());
m_backgroundRunning = false;
}
void SpecBase::cancelBackgroundIndex()
{
m_backgroundRunning = false;
if (m_backgroundIndexTh.joinable())
m_backgroundIndexTh.join();
}
void SpecBase::beginBackgroundIndex()
{
cancelBackgroundIndex();
clearTagCache();
m_backgroundRunning = true;
m_backgroundIndexTh = std::thread(std::bind(&SpecBase::backgroundIndexProc, this));
}
void SpecBase::waitForIndexComplete() const
{
std::unique_lock<std::mutex> lk(const_cast<SpecBase&>(*this).m_backgroundIndexMutex);
while (m_backgroundRunning)
{
lk.unlock();
std::this_thread::sleep_for(std::chrono::milliseconds(2));
lk.lock();
}
}
}

88
DataSpec/SpecBase.hpp
View File

@ -2,15 +2,14 @@
#define SPECBASE_HPP
#include <functional>
#include <thread>
#include <hecl/Database.hpp>
#include <nod/nod.hpp>
#include <athena/DNAYaml.hpp>
#include <athena/FileWriter.hpp>
#include "hecl/Blender/BlenderConnection.hpp"
namespace urde
{
struct SObjectTag;
}
#include "Runtime/RetroTypes.hpp"
namespace DataSpec
{
@ -29,8 +28,9 @@ struct SpecBase : hecl::Database::IDataSpec
void doCook(const hecl::ProjectPath& path, const hecl::ProjectPath& cookedPath,
bool fast, hecl::BlenderToken& btok, FCookProgress progress);
bool canPackage(const PackagePassInfo& info);
void doPackage(const PackagePassInfo& info);
bool canPackage(const hecl::ProjectPath& path);
void doPackage(const hecl::ProjectPath& path, const hecl::Database::DataSpecEntry* entry,
bool fast, hecl::BlenderToken& btok, FProgress progress, hecl::ClientProcess* cp);
/* Extract handlers */
virtual bool checkStandaloneID(const char* id) const=0;
@ -46,7 +46,7 @@ struct SpecBase : hecl::Database::IDataSpec
FProgress progress)=0;
/* Convert path to object tag */
virtual urde::SObjectTag BuildTagFromPath(const hecl::ProjectPath& path,
virtual urde::SObjectTag buildTagFromPath(const hecl::ProjectPath& path,
hecl::BlenderToken& btok) const=0;
/* Even if PC spec is being cooked, this will return the vanilla GCN spec */
@ -107,23 +107,93 @@ struct SpecBase : hecl::Database::IDataSpec
void flattenDependencies(const class UniqueID64& id, std::vector<hecl::ProjectPath>& pathsOut);
virtual void flattenDependenciesYAML(athena::io::IStreamReader& fin, std::vector<hecl::ProjectPath>& pathsOut)=0;
virtual void buildWorldPakList(const hecl::ProjectPath& worldPath,
const hecl::ProjectPath& worldPathCooked,
hecl::BlenderToken& btok,
athena::io::FileWriter& w,
std::vector<urde::SObjectTag>& listOut,
atUint64& resTableOffset) {}
virtual void buildPakList(hecl::BlenderToken& btok,
athena::io::FileWriter& w,
const std::vector<urde::SObjectTag>& list,
const std::vector<std::pair<urde::SObjectTag, std::string>>& nameList,
atUint64& resTableOffset) {}
virtual void writePakFileIndex(athena::io::FileWriter& w,
const std::vector<urde::SObjectTag>& tags,
const std::vector<std::tuple<size_t, size_t, bool>>& index,
atUint64 resTableOffset) {}
virtual std::pair<std::unique_ptr<uint8_t[]>, size_t>
compressPakData(const urde::SObjectTag& tag, const uint8_t* data, size_t len) { return {}; }
const hecl::ProjectPath& getMasterShaderPath() const {return m_masterShader;}
/* Support functions for resolving paths from IDs */
virtual hecl::ProjectPath getWorking(class UniqueID32&) {return hecl::ProjectPath();}
virtual hecl::ProjectPath getWorking(class UniqueID64&) {return hecl::ProjectPath();}
hecl::ProjectPath getCookedPath(const hecl::ProjectPath& working, bool pcTarget) const;
/* Project accessor */
hecl::Database::Project& getProject() const {return m_project;}
/* Extract RandomStatic entropy */
void ExtractRandomStaticEntropy(const uint8_t* buf, const hecl::ProjectPath& noAramPath);
void extractRandomStaticEntropy(const uint8_t* buf, const hecl::ProjectPath& noAramPath);
/* Tag cache functions */
urde::SObjectTag tagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const;
hecl::ProjectPath pathFromTag(const urde::SObjectTag& tag) const;
bool waitForTagReady(const urde::SObjectTag& tag, const hecl::ProjectPath*& pathOut);
const urde::SObjectTag* getResourceIdByName(const char* name) const;
hecl::FourCC getResourceTypeById(urde::CAssetId id) const;
void enumerateResources(const std::function<bool(const urde::SObjectTag&)>& lambda) const;
void enumerateNamedResources(
const std::function<bool(const std::string&, const urde::SObjectTag&)>& lambda) const;
void cancelBackgroundIndex();
void beginBackgroundIndex();
bool backgroundIndexRunning() const { return m_backgroundRunning; }
void waitForIndexComplete() const;
virtual void getTagListForFile(const char* pakName, std::vector<urde::SObjectTag>& out) const {}
SpecBase(const hecl::Database::DataSpecEntry* specEntry, hecl::Database::Project& project, bool pc);
~SpecBase();
protected:
hecl::Database::Project& m_project;
bool m_pc;
hecl::ProjectPath m_masterShader;
std::unordered_map<urde::SObjectTag, hecl::ProjectPath> m_tagToPath;
std::unordered_map<hecl::Hash, urde::SObjectTag> m_pathToTag;
std::unordered_map<std::string, urde::SObjectTag> m_catalogNameToTag;
std::unordered_map<urde::SObjectTag, std::string> m_catalogTagToName;
void clearTagCache();
hecl::BlenderToken m_backgroundBlender;
std::thread m_backgroundIndexTh;
std::mutex m_backgroundIndexMutex;
bool m_backgroundRunning = false;
void readCatalog(const hecl::ProjectPath& catalogPath,
athena::io::YAMLDocWriter& nameWriter);
bool addFileToIndex(const hecl::ProjectPath& path,
athena::io::YAMLDocWriter& cacheWriter);
void backgroundIndexRecursiveProc(const hecl::ProjectPath& path,
athena::io::YAMLDocWriter& cacheWriter,
athena::io::YAMLDocWriter& nameWriter,
int level);
void backgroundIndexRecursiveCatalogs(const hecl::ProjectPath& path,
athena::io::YAMLDocWriter& nameWriter,
int level);
void backgroundIndexProc();
void recursiveBuildResourceList(std::vector<urde::SObjectTag>& listOut,
const hecl::ProjectPath& path,
hecl::BlenderToken& btok);
void copyBuildListData(std::vector<std::tuple<size_t, size_t, bool>>& fileIndex,
const std::vector<urde::SObjectTag>& buildList,
const hecl::Database::DataSpecEntry* entry,
bool fast, FProgress progress, athena::io::FileWriter& pakOut);
private:
std::unique_ptr<nod::DiscBase> m_disc;
bool m_isWii;

220
DataSpec/SpecMP1.cpp
View File

@ -430,7 +430,7 @@ struct SpecMP1 : SpecBase
/* Extract part of .dol for RandomStatic entropy */
hecl::ProjectPath noAramPath(m_project.getProjectWorkingPath(), _S("MP1/NoARAM"));
ExtractRandomStaticEntropy(m_dolBuf.get() + 0x4f60, noAramPath);
extractRandomStaticEntropy(m_dolBuf.get() + 0x4f60, noAramPath);
return true;
}
@ -517,7 +517,7 @@ struct SpecMP1 : SpecBase
});
}
urde::SObjectTag BuildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
urde::SObjectTag buildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
{
if (hecl::StringUtils::EndsWith(path.getAuxInfo(), _S(".CINF")))
return {SBIG('CINF'), path.hash().val32()};
@ -718,6 +718,18 @@ struct SpecMP1 : SpecBase
return {};
}
void getTagListForFile(const char* pakName, std::vector<urde::SObjectTag>& out) const
{
std::string pathPrefix("MP1/");
pathPrefix += pakName;
pathPrefix += '/';
std::unique_lock<std::mutex> lk(const_cast<SpecMP1&>(*this).m_backgroundIndexMutex);
for (const auto& tag : m_tagToPath)
if (!tag.second.getRelativePathUTF8().compare(0, pathPrefix.size(), pathPrefix))
out.push_back(tag.first);
}
void cookMesh(const hecl::ProjectPath& out, const hecl::ProjectPath& in, BlendStream& ds, bool fast,
hecl::BlenderToken& btok, FCookProgress progress)
{
@ -1109,6 +1121,210 @@ struct SpecMP1 : SpecBase
}
}
void buildWorldPakList(const hecl::ProjectPath& worldPath,
const hecl::ProjectPath& worldPathCooked,
hecl::BlenderToken& btok,
athena::io::FileWriter& w,
std::vector<urde::SObjectTag>& listOut,
atUint64& resTableOffset)
{
DNAMP1::MLVL mlvl;
{
athena::io::FileReader r(worldPathCooked.getAbsolutePath());
if (r.hasError())
Log.report(logvisor::Fatal, _S("Unable to open world %s"), worldPathCooked.getRelativePath().c_str());
mlvl.read(r);
}
size_t count = 5;
for (const auto& area : mlvl.areas)
for (const auto& dep : area.deps)
++count;
listOut.reserve(count);
urde::SObjectTag worldTag = tagFromPath(worldPath.getWithExtension(_S(".*"), true), btok);
w.writeUint32Big(0x80030005);
w.writeUint32Big(0);
w.writeUint32Big(1);
DNAMP1::PAK::NameEntry nameEnt;
hecl::ProjectPath parentDir = worldPath.getParentPath();
nameEnt.type = worldTag.type;
nameEnt.id = atUint32(worldTag.id.Value());
nameEnt.nameLen = atUint32(hecl::StrLen(parentDir.getLastComponent()));
nameEnt.name = parentDir.getLastComponent();
nameEnt.write(w);
w.writeUint32Big(atUint32(count));
resTableOffset = w.position();
for (const auto& area : mlvl.areas)
for (const auto& dep : area.deps)
listOut.push_back({dep.type, dep.id.toUint32()});
urde::SObjectTag nameTag(FOURCC('STRG'), mlvl.worldNameId.toUint32());
if (nameTag)
listOut.push_back(nameTag);
urde::SObjectTag savwTag(FOURCC('SAVW'), mlvl.saveWorldId.toUint32());
if (savwTag)
{
if (hecl::ProjectPath savwPath = pathFromTag(savwTag))
m_project.cookPath(savwPath, {}, false, true);
listOut.push_back(savwTag);
}
urde::SObjectTag mapTag(FOURCC('MAPW'), mlvl.worldMap.toUint32());
if (mapTag)
{
if (hecl::ProjectPath mapPath = pathFromTag(mapTag))
{
m_project.cookPath(mapPath, {}, false, true);
if (hecl::ProjectPath mapCookedPath = getCookedPath(mapPath, true))
{
athena::io::FileReader r(mapCookedPath.getAbsolutePath());
if (r.hasError())
Log.report(logvisor::Fatal, _S("Unable to open %s"), mapCookedPath.getRelativePath().c_str());
if (r.readUint32Big() != 0xDEADF00D)
Log.report(logvisor::Fatal, _S("Corrupt MAPW %s"), mapCookedPath.getRelativePath().c_str());
r.readUint32Big();
atUint32 mapaCount = r.readUint32Big();
for (int i=0 ; i<mapaCount ; ++i)
{
UniqueID32 id;
id.read(r);
listOut.push_back({FOURCC('MAPA'), id.toUint32()});
}
}
}
listOut.push_back(mapTag);
}
urde::SObjectTag skyboxTag(FOURCC('CMDL'), mlvl.worldSkyboxId.toUint32());
if (skyboxTag)
{
hecl::ProjectPath skyboxPath = pathFromTag(skyboxTag);
if (btok.getBlenderConnection().openBlend(skyboxPath))
{
auto data = btok.getBlenderConnection().beginData();
std::vector<hecl::ProjectPath> textures = data.getTextures();
for (const auto& tex : textures)
{
urde::SObjectTag texTag = tagFromPath(tex, btok);
if (!texTag)
Log.report(logvisor::Fatal, _S("Unable to resolve %s"), tex.getRelativePath().c_str());
listOut.push_back(texTag);
}
}
listOut.push_back(skyboxTag);
}
listOut.push_back(worldTag);
for (const auto& item : listOut)
{
DNAMP1::PAK::Entry ent;
ent.compressed = 0;
ent.type = item.type;
ent.id = atUint32(item.id.Value());
ent.size = 0;
ent.offset = 0;
ent.write(w);
}
}
void buildPakList(hecl::BlenderToken& btok,
athena::io::FileWriter& w,
const std::vector<urde::SObjectTag>& list,
const std::vector<std::pair<urde::SObjectTag, std::string>>& nameList,
atUint64& resTableOffset)
{
w.writeUint32Big(0x80030005);
w.writeUint32Big(0);
w.writeUint32Big(atUint32(nameList.size()));
for (const auto& item : nameList)
{
DNAMP1::PAK::NameEntry nameEnt;
nameEnt.type = item.first.type;
nameEnt.id = atUint32(item.first.id.Value());
nameEnt.nameLen = atUint32(item.second.size());
nameEnt.name = item.second;
nameEnt.write(w);
}
w.writeUint32Big(atUint32(list.size()));
resTableOffset = w.position();
for (const auto& item : list)
{
DNAMP1::PAK::Entry ent;
ent.compressed = 0;
ent.type = item.type;
ent.id = atUint32(item.id.Value());
ent.size = 0;
ent.offset = 0;
ent.write(w);
}
}
void writePakFileIndex(athena::io::FileWriter& w,
const std::vector<urde::SObjectTag>& tags,
const std::vector<std::tuple<size_t, size_t, bool>>& index,
atUint64 resTableOffset)
{
w.seek(resTableOffset, athena::Begin);
auto it = tags.begin();
for (const auto& item : index)
{
const urde::SObjectTag& tag = *it++;
DNAMP1::PAK::Entry ent;
ent.compressed = atUint32(std::get<2>(item));
ent.type = tag.type;
ent.id = atUint32(tag.id.Value());
ent.size = atUint32(std::get<1>(item));
ent.offset = atUint32(std::get<0>(item));
ent.write(w);
}
}
std::pair<std::unique_ptr<uint8_t[]>, size_t>
compressPakData(const urde::SObjectTag& tag, const uint8_t* data, size_t len)
{
bool doCompress = false;
switch (tag.type)
{
case SBIG('TXTR'):
case SBIG('CMDL'):
case SBIG('CSKR'):
case SBIG('ANCS'):
case SBIG('ANIM'):
case SBIG('FONT'):
doCompress = true;
break;
case SBIG('PART'):
case SBIG('ELSC'):
case SBIG('SWHC'):
case SBIG('WPSC'):
case SBIG('DPSC'):
case SBIG('CRSC'):
doCompress = len >= 0x400;
break;
default:
break;
}
if (!doCompress)
return {};
uLong destLen = compressBound(len);
std::pair<std::unique_ptr<uint8_t[]>, size_t> ret;
ret.first.reset(new uint8_t[destLen]);
compress(ret.first.get(), &destLen, data, len);
ret.second = destLen;
return ret;
};
void cookAudioGroup(const hecl::ProjectPath& out, const hecl::ProjectPath& in, FCookProgress progress)
{
DNAMP1::AGSC::Cook(in, out);

2
DataSpec/SpecMP2.cpp
View File

@ -318,7 +318,7 @@ struct SpecMP2 : SpecBase
});
}
urde::SObjectTag BuildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
urde::SObjectTag buildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
{
return {};
}

2
DataSpec/SpecMP3.cpp
View File

@ -512,7 +512,7 @@ struct SpecMP3 : SpecBase
return false;
}
urde::SObjectTag BuildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
urde::SObjectTag buildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
{
return {};
}

546
Editor/ProjectResourceFactoryBase.cpp
View File

@ -1,445 +1,21 @@
#include "ProjectResourceFactoryBase.hpp"
#include "Runtime/IObj.hpp"
#define DUMP_CACHE_FILL 1
namespace urde
{
static logvisor::Module Log("urde::ProjectResourceFactoryBase");
static void WriteTag(athena::io::YAMLDocWriter& cacheWriter,
const SObjectTag& pathTag, const hecl::ProjectPath& path)
{
char idStr[9];
snprintf(idStr, 9, "%08X", uint32_t(pathTag.id.Value()));
if (auto v = cacheWriter.enterSubVector(idStr))
{
cacheWriter.writeString(nullptr, pathTag.type.toString().c_str());
cacheWriter.writeString(nullptr, path.getAuxInfo().size() ?
(path.getRelativePathUTF8() + '|' + path.getAuxInfoUTF8()) :
path.getRelativePathUTF8());
}
}
static void WriteNameTag(athena::io::YAMLDocWriter& nameWriter,
const SObjectTag& pathTag,
const std::string& name)
{
char idStr[9];
snprintf(idStr, 9, "%08X", uint32_t(pathTag.id.Value()));
nameWriter.writeString(name.c_str(), idStr);
}
void ProjectResourceFactoryBase::Clear()
{
m_tagToPath.clear();
m_pathToTag.clear();
m_catalogNameToTag.clear();
}
SObjectTag ProjectResourceFactoryBase::TagFromPath(const hecl::ProjectPath& path,
hecl::BlenderToken& btok) const
{
auto search = m_pathToTag.find(path.hash());
if (search != m_pathToTag.cend())
return search->second;
return BuildTagFromPath(path, btok);
}
void ProjectResourceFactoryBase::ReadCatalog(const hecl::ProjectPath& catalogPath,
athena::io::YAMLDocWriter& nameWriter)
{
athena::io::FileReader freader(catalogPath.getAbsolutePath());
if (!freader.isOpen())
return;
athena::io::YAMLDocReader reader;
bool res = reader.parse(&freader);
if (!res)
return;
const athena::io::YAMLNode* root = reader.getRootNode();
for (const auto& p : root->m_mapChildren)
{
/* Hash as lowercase since lookup is case-insensitive */
std::string pLower = p.first;
std::transform(pLower.cbegin(), pLower.cend(), pLower.begin(), tolower);
/* Avoid redundant filesystem access for re-caches */
if (m_catalogNameToTag.find(pLower) != m_catalogNameToTag.cend())
continue;
athena::io::YAMLNode& node = *p.second;
hecl::ProjectPath path;
if (node.m_type == YAML_SCALAR_NODE)
{
path = hecl::ProjectPath(m_proj->getProjectWorkingPath(), node.m_scalarString);
}
else if (node.m_type == YAML_SEQUENCE_NODE)
{
if (node.m_seqChildren.size() >= 2)
path = hecl::ProjectPath(m_proj->getProjectWorkingPath(), node.m_seqChildren[0]->m_scalarString).
ensureAuxInfo(node.m_seqChildren[1]->m_scalarString);
else if (node.m_seqChildren.size() == 1)
path = hecl::ProjectPath(m_proj->getProjectWorkingPath(), node.m_seqChildren[0]->m_scalarString);
}
if (!path.isFileOrGlob())
continue;
SObjectTag pathTag = TagFromPath(path, m_backgroundBlender);
if (pathTag)
{
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
m_catalogNameToTag[pLower] = pathTag;
WriteNameTag(nameWriter, pathTag, p.first);
#if 0
fprintf(stderr, "%s %s %08X\n",
p.first.c_str(),
pathTag.type.toString().c_str(), uint32_t(pathTag.id));
#endif
}
}
}
void ProjectResourceFactoryBase::BackgroundIndexRecursiveCatalogs(const hecl::ProjectPath& dir,
athena::io::YAMLDocWriter& nameWriter,
int level)
{
hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(),
hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted,
false, false, true);
/* Enumerate all items */
for (const hecl::DirectoryEnumerator::Entry& ent : dEnum)
{
hecl::ProjectPath path(dir, ent.m_name);
if (ent.m_isDir && level < 1)
BackgroundIndexRecursiveCatalogs(path, nameWriter, level+1);
else
{
if (!path.isFile())
continue;
/* Read catalog.yaml for .pak directory if exists */
if (level == 1 && !ent.m_name.compare(_S("!catalog.yaml")))
{
ReadCatalog(path, nameWriter);
continue;
}
}
/* bail if cancelled by client */
if (!m_backgroundRunning)
break;
}
}
#if DUMP_CACHE_FILL
static void DumpCacheAdd(const SObjectTag& pathTag, const hecl::ProjectPath& path)
{
fprintf(stderr, "%s %08X %s\n",
pathTag.type.toString().c_str(), uint32_t(pathTag.id.Value()),
path.getRelativePathUTF8().c_str());
}
#endif
bool ProjectResourceFactoryBase::AddFileToIndex(const hecl::ProjectPath& path,
athena::io::YAMLDocWriter& cacheWriter)
{
/* Avoid redundant filesystem access for re-caches */
if (m_pathToTag.find(path.hash()) != m_pathToTag.cend())
return true;
/* Try as glob */
hecl::ProjectPath asGlob = path.getWithExtension(_S(".*"), true);
if (m_pathToTag.find(asGlob.hash()) != m_pathToTag.cend())
return true;
/* Classify intermediate into tag */
SObjectTag pathTag = BuildTagFromPath(path, m_backgroundBlender);
if (pathTag)
{
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
bool useGlob = false;
/* Special multi-resource intermediates */
if (pathTag.type == SBIG('ANCS'))
{
hecl::BlenderConnection& conn = m_backgroundBlender.getBlenderConnection();
if (!conn.openBlend(path) || conn.getBlendType() != hecl::BlenderConnection::BlendType::Actor)
return false;
/* Transform tag to glob */
pathTag = {SBIG('ANCS'), asGlob.hash().val32()};
useGlob = true;
hecl::BlenderConnection::DataStream ds = conn.beginData();
std::vector<std::string> armatureNames = ds.getArmatureNames();
std::vector<std::string> subtypeNames = ds.getSubtypeNames();
std::vector<std::string> actionNames = ds.getActionNames();
for (const std::string& arm : armatureNames)
{
hecl::SystemStringView sysStr(arm);
hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".CINF"));
SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
m_tagToPath[pathTag] = subPath;
m_pathToTag[subPath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, subPath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, subPath);
#endif
}
for (const std::string& sub : subtypeNames)
{
hecl::SystemStringView sysStr(sub);
hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".CSKR"));
SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
m_tagToPath[pathTag] = subPath;
m_pathToTag[subPath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, subPath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, subPath);
#endif
}
for (const std::string& act : actionNames)
{
hecl::SystemStringView sysStr(act);
hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".ANIM"));
SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
m_tagToPath[pathTag] = subPath;
m_pathToTag[subPath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, subPath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, subPath);
#endif
}
}
else if (pathTag.type == SBIG('MLVL'))
{
/* Transform tag to glob */
pathTag = {SBIG('MLVL'), asGlob.hash().val32()};
useGlob = true;
hecl::ProjectPath subPath = asGlob.ensureAuxInfo(_S("MAPW"));
SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
m_tagToPath[pathTag] = subPath;
m_pathToTag[subPath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, subPath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, subPath);
#endif
subPath = asGlob.ensureAuxInfo(_S("SAVW"));
pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
m_tagToPath[pathTag] = subPath;
m_pathToTag[subPath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, subPath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, subPath);
#endif
}
else if (pathTag.type == SBIG('AGSC'))
{
/* Transform tag to glob */
pathTag = {SBIG('AGSC'), asGlob.hash().val32()};
useGlob = true;
}
else if (pathTag.type == SBIG('MREA'))
{
hecl::ProjectPath subPath = path.ensureAuxInfo(_S("PATH"));
SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
m_tagToPath[pathTag] = subPath;
m_pathToTag[subPath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, subPath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, subPath);
#endif
}
/* Cache in-memory */
const hecl::ProjectPath& usePath = useGlob ? asGlob : path;
m_tagToPath[pathTag] = usePath;
m_pathToTag[usePath.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, usePath);
#if DUMP_CACHE_FILL
DumpCacheAdd(pathTag, usePath);
#endif
}
return true;
}
void ProjectResourceFactoryBase::BackgroundIndexRecursiveProc(const hecl::ProjectPath& dir,
athena::io::YAMLDocWriter& cacheWriter,
athena::io::YAMLDocWriter& nameWriter,
int level)
{
hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(),
hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted,
false, false, true);
/* Enumerate all items */
for (const hecl::DirectoryEnumerator::Entry& ent : dEnum)
{
hecl::ProjectPath path(dir, ent.m_name);
if (ent.m_isDir)
BackgroundIndexRecursiveProc(path, cacheWriter, nameWriter, level+1);
else
{
if (!path.isFile())
continue;
/* Read catalog.yaml for .pak directory if exists */
if (level == 1 && !ent.m_name.compare(_S("!catalog.yaml")))
{
ReadCatalog(path, nameWriter);
continue;
}
/* Index the regular file */
AddFileToIndex(path, cacheWriter);
}
/* bail if cancelled by client */
if (!m_backgroundRunning)
break;
}
}
void ProjectResourceFactoryBase::BackgroundIndexProc()
{
logvisor::RegisterThreadName("Resource Index Thread");
hecl::ProjectPath tagCachePath(m_proj->getProjectCookedPath(*m_origSpec), _S("tag_cache.yaml"));
hecl::ProjectPath nameCachePath(m_proj->getProjectCookedPath(*m_origSpec), _S("name_cache.yaml"));
hecl::ProjectPath specRoot(m_proj->getProjectWorkingPath(), m_origSpec->m_name);
/* Cache will be overwritten with validated entries afterwards */
athena::io::YAMLDocWriter cacheWriter(nullptr);
athena::io::YAMLDocWriter nameWriter(nullptr);
/* Read in tag cache */
if (tagCachePath.isFile())
{
athena::io::FileReader reader(tagCachePath.getAbsolutePath());
if (reader.isOpen())
{
Log.report(logvisor::Info, _S("Cache index of '%s' loading"), m_origSpec->m_name);
athena::io::YAMLDocReader cacheReader;
if (cacheReader.parse(&reader))
{
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
size_t tagCount = cacheReader.getRootNode()->m_mapChildren.size();
m_tagToPath.reserve(tagCount);
m_pathToTag.reserve(tagCount);
size_t loadIdx = 0;
for (const auto& child : cacheReader.getRootNode()->m_mapChildren)
{
const athena::io::YAMLNode& node = *child.second;
unsigned long id = strtoul(child.first.c_str(), nullptr, 16);
hecl::FourCC type(node.m_seqChildren.at(0)->m_scalarString.c_str());
hecl::ProjectPath path(m_proj->getProjectWorkingPath(),
node.m_seqChildren.at(1)->m_scalarString);
if (path.isFileOrGlob())
{
SObjectTag pathTag(type, id);
m_tagToPath[pathTag] = path;
m_pathToTag[path.hash()] = pathTag;
WriteTag(cacheWriter, pathTag, path);
}
fprintf(stderr, "\r %" PRISize " / %" PRISize, ++loadIdx, tagCount);
}
fprintf(stderr, "\n");
}
Log.report(logvisor::Info, _S("Cache index of '%s' loaded; %d tags"),
m_origSpec->m_name, m_tagToPath.size());
if (nameCachePath.isFile())
{
/* Read in name cache */
Log.report(logvisor::Info, _S("Name index of '%s' loading"), m_origSpec->m_name);
athena::io::FileReader nreader(nameCachePath.getAbsolutePath());
athena::io::YAMLDocReader nameReader;
if (nameReader.parse(&nreader))
{
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
m_catalogNameToTag.reserve(nameReader.getRootNode()->m_mapChildren.size());
for (const auto& child : nameReader.getRootNode()->m_mapChildren)
{
unsigned long id = strtoul(child.second->m_scalarString.c_str(), nullptr, 16);
auto search = m_tagToPath.find(SObjectTag(FourCC(), uint32_t(id)));
if (search != m_tagToPath.cend())
{
std::string chLower = child.first;
std::transform(chLower.cbegin(), chLower.cend(), chLower.begin(), tolower);
m_catalogNameToTag[chLower] = search->first;
WriteNameTag(nameWriter, search->first, child.first);
}
}
}
Log.report(logvisor::Info, _S("Name index of '%s' loaded; %d names"),
m_origSpec->m_name, m_catalogNameToTag.size());
}
}
}
/* Add special original IDs resource if exists (not name-cached to disk) */
hecl::ProjectPath oidsPath(specRoot, "!original_ids.yaml");
SObjectTag oidsTag = BuildTagFromPath(oidsPath, m_backgroundBlender);
if (oidsTag)
m_catalogNameToTag["mp1originalids"] = oidsTag;
Log.report(logvisor::Info, _S("Background index of '%s' started"), m_origSpec->m_name);
BackgroundIndexRecursiveProc(specRoot, cacheWriter, nameWriter, 0);
tagCachePath.makeDirChain(false);
athena::io::FileWriter twriter(tagCachePath.getAbsolutePath());
cacheWriter.finish(&twriter);
athena::io::FileWriter nwriter(nameCachePath.getAbsolutePath());
nameWriter.finish(&nwriter);
m_backgroundBlender.shutdown();
Log.report(logvisor::Info, _S("Background index of '%s' complete; %d tags, %d names"),
m_origSpec->m_name, m_tagToPath.size(), m_catalogNameToTag.size());
m_backgroundRunning = false;
}
void ProjectResourceFactoryBase::CancelBackgroundIndex()
{
m_backgroundRunning = false;
if (m_backgroundIndexTh.joinable())
m_backgroundIndexTh.join();
}
void ProjectResourceFactoryBase::BeginBackgroundIndex
(hecl::Database::Project& proj,
const hecl::Database::DataSpecEntry& origSpec,
const hecl::Database::DataSpecEntry& pcSpec)
{
CancelBackgroundIndex();
Clear();
m_proj = &proj;
m_origSpec = &origSpec;
m_pcSpec = &pcSpec;
m_cookSpec.reset(pcSpec.m_factory(proj, hecl::Database::DataSpecTool::Cook));
m_backgroundRunning = true;
m_backgroundIndexTh =
std::thread(std::bind(&ProjectResourceFactoryBase::BackgroundIndexProc, this));
}
hecl::ProjectPath ProjectResourceFactoryBase::GetCookedPath(const hecl::ProjectPath& working,
bool pcTarget) const
{
const hecl::Database::DataSpecEntry* spec = m_origSpec;
if (pcTarget)
spec = m_cookSpec->overrideDataSpec(working, m_pcSpec, hecl::SharedBlenderToken);
if (!spec)
return {};
return working.getCookedPath(*spec);
return static_cast<DataSpec::SpecBase&>(*m_cookSpec).beginBackgroundIndex();
}
bool ProjectResourceFactoryBase::SyncCook(const hecl::ProjectPath& working)
@ -629,35 +205,6 @@ ProjectResourceFactoryBase::_RemoveTask(const SObjectTag& tag)
return _RemoveTask(m_asyncLoadMap.find(tag));
};
bool ProjectResourceFactoryBase::WaitForTagReady(const urde::SObjectTag& tag,
const hecl::ProjectPath*& pathOut)
{
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
auto search = m_tagToPath.find(tag);
if (search == m_tagToPath.end())
{
if (m_backgroundRunning)
{
while (m_backgroundRunning)
{
lk.unlock();
std::this_thread::sleep_for(std::chrono::milliseconds(2));
lk.lock();
search = m_tagToPath.find(tag);
if (search != m_tagToPath.end())
break;
}
if (search == m_tagToPath.end())
return false;
}
else
return false;
}
lk.unlock();
pathOut = &search->second;
return true;
}
bool
ProjectResourceFactoryBase::PrepForReadSync(const SObjectTag& tag,
const hecl::ProjectPath& path,
@ -922,108 +469,34 @@ bool ProjectResourceFactoryBase::CanBuild(const urde::SObjectTag& tag)
const urde::SObjectTag* ProjectResourceFactoryBase::GetResourceIdByName(const char* name) const
{
std::string lower = name;
std::transform(lower.cbegin(), lower.cend(), lower.begin(), tolower);
std::unique_lock<std::mutex> lk(const_cast<ProjectResourceFactoryBase*>(this)->m_backgroundIndexMutex);
auto search = m_catalogNameToTag.find(lower);
if (search == m_catalogNameToTag.end())
{
if (m_backgroundRunning)
{
while (m_backgroundRunning)
{
lk.unlock();
std::this_thread::sleep_for(std::chrono::milliseconds(2));
lk.lock();
search = m_catalogNameToTag.find(lower);
if (search != m_catalogNameToTag.end())
break;
}
if (search == m_catalogNameToTag.end())
return nullptr;
}
else
return nullptr;
}
return &search->second;
return static_cast<DataSpec::SpecBase&>(*m_cookSpec).getResourceIdByName(name);
}
FourCC ProjectResourceFactoryBase::GetResourceTypeById(CAssetId id) const
{
if (!id.IsValid())
return {};
std::unique_lock<std::mutex> lk(const_cast<ProjectResourceFactoryBase*>(this)->m_backgroundIndexMutex);
SObjectTag searchTag = {FourCC(), id};
auto search = m_tagToPath.find(searchTag);
if (search == m_tagToPath.end())
{
if (m_backgroundRunning)
{
while (m_backgroundRunning)
{
lk.unlock();
std::this_thread::sleep_for(std::chrono::milliseconds(2));
lk.lock();
search = m_tagToPath.find(searchTag);
if (search != m_tagToPath.end())
break;
}
if (search == m_tagToPath.end())
return {};
}
else
return {};
}
return search->first.type;
return static_cast<DataSpec::SpecBase&>(*m_cookSpec).getResourceTypeById(id);
}
void ProjectResourceFactoryBase::EnumerateResources(const std::function<bool(const SObjectTag&)>& lambda) const
{
std::unique_lock<std::mutex> lk(const_cast<ProjectResourceFactoryBase*>(this)->m_backgroundIndexMutex);
while (m_backgroundRunning)
{
lk.unlock();
std::this_thread::sleep_for(std::chrono::milliseconds(2));
lk.lock();
}
for (const auto& pair : m_tagToPath)
{
if (!lambda(pair.first))
break;
}
return static_cast<DataSpec::SpecBase&>(*m_cookSpec).enumerateResources(lambda);
}
void ProjectResourceFactoryBase::EnumerateNamedResources(
const std::function<bool(const std::string&, const SObjectTag&)>& lambda) const
{
std::unique_lock<std::mutex> lk(const_cast<ProjectResourceFactoryBase*>(this)->m_backgroundIndexMutex);
while (m_backgroundRunning)
{
lk.unlock();
std::this_thread::sleep_for(std::chrono::milliseconds(2));
lk.lock();
}
lk.unlock();
for (const auto& pair : m_catalogNameToTag)
{
if (!lambda(pair.first, pair.second))
break;
}
return static_cast<DataSpec::SpecBase&>(*m_cookSpec).enumerateNamedResources(lambda);
}
template <typename ItType>
bool ProjectResourceFactoryBase::AsyncPumpTask(ItType& it)
{
/* Ensure requested resource is in the index */
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
AsyncTask& task = _GetAsyncTask(it);
auto search = m_tagToPath.find(task.x0_tag);
if (search == m_tagToPath.end())
hecl::ProjectPath path = static_cast<DataSpec::SpecBase&>(*m_cookSpec).pathFromTag(task.x0_tag);
if (!path)
{
if (!m_backgroundRunning)
if (!static_cast<DataSpec::SpecBase&>(*m_cookSpec).backgroundIndexRunning())
{
Log.report(logvisor::Error, _S("unable to find async load resource (%s, %08X)"),
task.x0_tag.type.toString().c_str(), task.x0_tag.id);
@ -1031,8 +504,7 @@ bool ProjectResourceFactoryBase::AsyncPumpTask(ItType& it)
}
return true;
}
lk.unlock();
task.EnsurePath(task.x0_tag, search->second);
task.EnsurePath(task.x0_tag, path);
/* Pump load pipeline (cooking if needed) */
if (task.AsyncPump())

55
Editor/ProjectResourceFactoryBase.hpp
View File

@ -6,6 +6,7 @@
#include "Runtime/IFactory.hpp"
#include "Runtime/CFactoryMgr.hpp"
#include "Runtime/CResFactory.hpp"
#include "DataSpec/SpecBase.hpp"
#include "optional.hpp"
#include <thread>
@ -72,11 +73,6 @@ public:
};
protected:
std::unordered_map<urde::SObjectTag, hecl::ProjectPath> m_tagToPath;
std::unordered_map<hecl::Hash, urde::SObjectTag> m_pathToTag;
std::unordered_map<std::string, urde::SObjectTag> m_catalogNameToTag;
void Clear();
const hecl::Database::Project* m_proj = nullptr;
const hecl::Database::DataSpecEntry* m_origSpec = nullptr;
const hecl::Database::DataSpecEntry* m_pcSpec = nullptr;
@ -84,11 +80,6 @@ protected:
std::unique_ptr<hecl::Database::IDataSpec> m_cookSpec;
urde::CFactoryMgr m_factoryMgr;
hecl::BlenderToken m_backgroundBlender;
std::thread m_backgroundIndexTh;
std::mutex m_backgroundIndexMutex;
bool m_backgroundRunning = false;
std::list<std::shared_ptr<AsyncTask>> m_asyncLoadList;
std::unordered_map<SObjectTag, std::list<std::shared_ptr<AsyncTask>>::iterator> m_asyncLoadMap;
std::shared_ptr<AsyncTask>
@ -111,36 +102,36 @@ protected:
return **it->second;
}
bool WaitForTagReady(const urde::SObjectTag& tag, const hecl::ProjectPath*& pathOut);
bool
PrepForReadSync(const SObjectTag& tag,
const hecl::ProjectPath& path,
std::experimental::optional<athena::io::FileReader>& fr);
SObjectTag TagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const;
bool WaitForTagReady(const urde::SObjectTag& tag, const hecl::ProjectPath*& pathOut)
{
return static_cast<DataSpec::SpecBase&>(*m_cookSpec).waitForTagReady(tag, pathOut);
}
SObjectTag TagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
{
return static_cast<DataSpec::SpecBase&>(*m_cookSpec).tagFromPath(path, btok);
}
SObjectTag BuildTagFromPath(const hecl::ProjectPath& path, hecl::BlenderToken& btok) const
{
return static_cast<DataSpec::SpecBase&>(*m_cookSpec).BuildTagFromPath(path, btok);
return static_cast<DataSpec::SpecBase&>(*m_cookSpec).buildTagFromPath(path, btok);
}
void GetTagListForFile(const char* pakName, std::vector<SObjectTag>& out) const
{
return static_cast<DataSpec::SpecBase&>(*m_cookSpec).getTagListForFile(pakName, out);
}
void CancelBackgroundIndex()
{
if (m_cookSpec)
return static_cast<DataSpec::SpecBase&>(*m_cookSpec).cancelBackgroundIndex();
}
void ReadCatalog(const hecl::ProjectPath& catalogPath,
athena::io::YAMLDocWriter& nameWriter);
bool AddFileToIndex(const hecl::ProjectPath& path,
athena::io::YAMLDocWriter& cacheWriter);
void BackgroundIndexRecursiveProc(const hecl::ProjectPath& path,
athena::io::YAMLDocWriter& cacheWriter,
athena::io::YAMLDocWriter& nameWriter,
int level);
void BackgroundIndexRecursiveCatalogs(const hecl::ProjectPath& path,
athena::io::YAMLDocWriter& nameWriter,
int level);
void BackgroundIndexProc();
void CancelBackgroundIndex();
void BeginBackgroundIndex(hecl::Database::Project& proj,
const hecl::Database::DataSpecEntry& origSpec,
const hecl::Database::DataSpecEntry& pcSpec);
const hecl::Database::DataSpecEntry& origSpec,
const hecl::Database::DataSpecEntry& pcSpec);
hecl::ProjectPath GetCookedPath(const hecl::ProjectPath& working, bool pcTarget) const;
bool SyncCook(const hecl::ProjectPath& working);
CFactoryFnReturn BuildSync(const SObjectTag& tag, const hecl::ProjectPath& path,
const CVParamTransfer& paramXfer, CObjectReference* selfRef);
@ -154,6 +145,10 @@ public:
bool CanBuild(const urde::SObjectTag&);
const urde::SObjectTag* GetResourceIdByName(const char*) const;
FourCC GetResourceTypeById(CAssetId id) const;
hecl::ProjectPath GetCookedPath(const hecl::ProjectPath& working, bool pcTarget) const
{
return static_cast<DataSpec::SpecBase&>(*m_cookSpec).getCookedPath(working, pcTarget);
}
void EnumerateResources(const std::function<bool(const SObjectTag&)>& lambda) const;
void EnumerateNamedResources(const std::function<bool(const std::string&, const SObjectTag&)>& lambda) const;

13
Editor/ProjectResourceFactoryMP1.cpp
View File

@ -135,19 +135,6 @@ void ProjectResourceFactoryMP1::IndexMP1Resources(hecl::Database::Project& proj,
m_origIds = sp.GetObj("MP1OriginalIDs");
}
void ProjectResourceFactoryMP1::GetTagListForFile(const char* pakName, std::vector<SObjectTag>& out) const
{
std::string pathPrefix("MP1/");
pathPrefix += pakName;
pathPrefix += '/';
std::unique_lock<std::mutex> lk(
const_cast<ProjectResourceFactoryMP1&>(*this).m_backgroundIndexMutex);
for (const auto& tag : m_tagToPath)
if (!tag.second.getRelativePathUTF8().compare(0, pathPrefix.size(), pathPrefix))
out.push_back(tag.first);
}
void ProjectResourceFactoryMP1::Shutdown()
{
m_origIds = TLockedToken<MP1OriginalIDs>();

1
Editor/ProjectResourceFactoryMP1.hpp
View File

@ -15,7 +15,6 @@ class ProjectResourceFactoryMP1 : public ProjectResourceFactoryBase
public:
ProjectResourceFactoryMP1(hecl::ClientProcess& clientProc);
void IndexMP1Resources(hecl::Database::Project& proj, CSimplePool& sp);
void GetTagListForFile(const char* pakName, std::vector<SObjectTag>& out) const;
void Shutdown();
CAssetId TranslateOriginalToNew(CAssetId id) const;

1
Runtime/CDvdFile.hpp
View File

@ -2,6 +2,7 @@
#define __URDE_CDVDFILE_HPP__
#include "RetroTypes.hpp"
#include "athena/FileReader.hpp"
#include <thread>
#include <mutex>

2
Runtime/CSortedLists.hpp
View File

@ -20,7 +20,7 @@ enum ESortedList
struct SSortedList
{
s16 x0_ids[1024];
u32 x800_size;
u32 x800_size = 0;
void Reset() {std::fill(std::begin(x0_ids), std::end(x0_ids), -1);}
SSortedList() {Reset();}
};

15
Runtime/CStaticInterference.cpp
View File

@ -1,4 +1,5 @@
#include "CStaticInterference.hpp"
#include "zeus/Math.hpp"
namespace urde
{
@ -24,7 +25,7 @@ void CStaticInterference::Update(CStateManager&, float dt)
newSources.reserve(m_sources.size());
for (CStaticInterferenceSource& src : m_sources)
{
if (src.timeLeft >= 0.0)
if (src.timeLeft >= 0.f)
{
src.timeLeft -= dt;
newSources.push_back(src);
@ -35,8 +36,8 @@ void CStaticInterference::Update(CStateManager&, float dt)
float CStaticInterference::GetTotalInterference() const
{
float validAccum = 0.0;
float invalidAccum = 0.0;
float validAccum = 0.f;
float invalidAccum = 0.f;
for (const CStaticInterferenceSource& src : m_sources)
{
if (src.id == kInvalidUniqueId)
@ -44,11 +45,11 @@ float CStaticInterference::GetTotalInterference() const
else
validAccum += src.magnitude;
}
if (validAccum > 0.80000001)
validAccum = 0.80000001;
if (validAccum > 0.80000001f)
validAccum = 0.80000001f;
validAccum += invalidAccum;
if (validAccum > 1.0)
return 1.0;
if (validAccum > 1.f)
return 1.f;
return validAccum;
}

1
Runtime/Camera/CCameraSpline.hpp
View File

@ -2,6 +2,7 @@
#define __URDE_CCAMERASPLINE_HPP__
#include "World/CEntityInfo.hpp"
#include "zeus/CVector3f.hpp"
namespace urde
{

1
Runtime/Character/CBodyStateCmdMgr.cpp
View File

@ -1,4 +1,5 @@
#include "CBodyStateCmdMgr.hpp"
#include <cfloat>
namespace urde
{

1
Runtime/GuiSys/CHudBossEnergyInterface.hpp
View File

@ -2,6 +2,7 @@
#define __URDE_CHUDBOSSENERGYINTERFACE_HPP__
#include "RetroTypes.hpp"
#include "zeus/CVector3f.hpp"
namespace urde
{

1
Runtime/GuiSys/CHudEnergyInterface.hpp
View File

@ -3,6 +3,7 @@
#include "RetroTypes.hpp"
#include "CHudInterface.hpp"
#include "zeus/CVector3f.hpp"
namespace urde
{

1
Runtime/GuiSys/CHudVisorBeamMenu.hpp
View File

@ -2,6 +2,7 @@
#define __URDE_CHUDVISORBEAMMENU_HPP__
#include "RetroTypes.hpp"
#include <cfloat>
namespace urde
{

1
Runtime/MP1/CNESEmulator.hpp
View File

@ -2,6 +2,7 @@
#define __URDE_CNESEMULATOR_HPP__
#include "RetroTypes.hpp"
#include "zeus/CColor.hpp"
namespace urde
{

31
Runtime/RetroTypes.hpp
View File

@ -4,10 +4,11 @@
#include <vector>
#include <utility>
#include <string>
#include <functional>
#include "GCNTypes.hpp"
#include "rstl.hpp"
#include "DataSpec/DNACommon/DNACommon.hpp"
#include "IOStreams.hpp"
#include "hecl/hecl.hpp"
namespace urde
{
@ -24,10 +25,9 @@ public:
explicit CAssetId(CInputStream& in);
bool IsValid() const { return id != UINT64_MAX; }
u64 Value() const { return id; }
void Assign(u64 v) { id = (v == UINT32_MAX ? UINT64_MAX : (v == 0 ? UINT64_MAX : v)); }
void Assign(u64 v) { id = (v == UINT32_MAX ? UINT64_MAX : (v == 0 ? UINT64_MAX : v)); }
void Reset() { id = UINT64_MAX; }
void PutTo(COutputStream& out);
bool operator==(const CAssetId& other) const { return id == other.id; }
bool operator!=(const CAssetId& other) const { return id != other.id; }
bool operator<(const CAssetId& other) const { return id < other.id; }
@ -39,6 +39,7 @@ struct SObjectTag
{
FourCC type;
CAssetId id;
operator bool() const { return id.IsValid(); }
bool operator!=(const SObjectTag& other) const { return id != other.id; }
bool operator==(const SObjectTag& other) const { return id == other.id; }
@ -65,11 +66,11 @@ struct TEditorId
u8 LayerNum() const { return u8((id >> 26) & 0x3f); }
u16 AreaNum() const { return u16((id >> 16) & 0x3ff); }
u16 Id() const { return u16(id & 0xffff); }
bool operator<(const TEditorId& other) const { return (id & 0x3ffffff) < (other.id & 0x3ffffff); }
bool operator!=(const TEditorId& other) const { return (id & 0x3ffffff) != (other.id & 0x3ffffff); }
bool operator==(const TEditorId& other) const { return (id & 0x3ffffff) == (other.id & 0x3ffffff); }
};
#define kInvalidEditorId TEditorId()
struct TUniqueId
@ -77,9 +78,8 @@ struct TUniqueId
TUniqueId() = default;
TUniqueId(u16 value, u16 version) : id(value | (version << 10)) {}
u16 id = u16(-1);
u16 Version() const { return u16((id >> 10) & 0x3f);}
u16 Value() const { return u16(id & 0x3ff);}
u16 Version() const { return u16((id >> 10) & 0x3f); }
u16 Value() const { return u16(id & 0x3ff); }
bool operator<(const TUniqueId& other) const { return (id < other.id); }
bool operator!=(const TUniqueId& other) const { return (id != other.id); }
bool operator==(const TUniqueId& other) const { return (id == other.id); }
@ -90,7 +90,6 @@ struct TUniqueId
using TAreaId = s32;
#define kInvalidAreaId TAreaId(-1)
}
#if 0
template <class T, size_t N>
@ -113,7 +112,7 @@ public:
};
#endif
template <class T>
template<class T>
T GetAverage(const T* v, s32 count)
{
T r = v[0];
@ -123,11 +122,12 @@ T GetAverage(const T* v, s32 count)
return r / count;
}
template <class T, size_t N>
template<class T, size_t N>
class TReservedAverage : rstl::reserved_vector<T, N>
{
public:
TReservedAverage() = default;
TReservedAverage(const T& t) { rstl::reserved_vector<T, N>::resize(N, t); }
void AddValue(const T& t)
@ -135,8 +135,7 @@ public:
if (this->size() < N)
{
this->insert(this->begin(), t);
}
else
} else
{
this->pop_back();
this->insert(this->begin(), t);
@ -148,7 +147,7 @@ public:
if (this->empty())
return {};
return {::GetAverage<T>(this->data(), this->size())};
return {urde::GetAverage<T>(this->data(), this->size())};
}
rstl::optional_object<T> GetEntry(int i) const
@ -163,18 +162,20 @@ public:
size_t Size() const { return this->size(); }
};
}
namespace std
{
template <>
struct hash<urde::SObjectTag>
{
inline size_t operator()(const urde::SObjectTag& tag) const { return tag.id.Value(); }
size_t operator()(const urde::SObjectTag& tag) const noexcept { return tag.id.Value(); }
};
template <>
struct hash<urde::CAssetId>
{
inline size_t operator()(const urde::CAssetId& id) const { return id.Value(); }
size_t operator()(const urde::CAssetId& id) const noexcept { return id.Value(); }
};
}

1
Runtime/World/CAiFuncMap.hpp
View File

@ -2,6 +2,7 @@
#define __URDE_CAIFUNCMAP_HPP__
#include "RetroTypes.hpp"
#include <unordered_map>
namespace urde
{

1
Runtime/World/CVisorFlare.hpp
View File

@ -3,6 +3,7 @@
#include "RetroTypes.hpp"
#include "CToken.hpp"
#include "zeus/CColor.hpp"
namespace urde
{

1
Runtime/World/CWorldLight.cpp
View File

@ -1,4 +1,5 @@
#include "CWorldLight.hpp"
#include <cfloat>
namespace urde
{

2
hecl

@ -1 +1 @@
Subproject commit 23a08dcf40456c59ff40710d12241a8a8735fe5e
Subproject commit 70b73855bf906c81d18bdf70c9b701bba996314e