mirror of https://github.com/AxioDL/metaforce.git
1055 lines
39 KiB
C++
1055 lines
39 KiB
C++
#include "ProjectResourceFactoryBase.hpp"
|
|
#include "Runtime/IObj.hpp"
|
|
|
|
#define DUMP_CACHE_FILL 1
|
|
|
|
namespace urde
|
|
{
|
|
static logvisor::Module Log("urde::ProjectResourceFactoryBase");
|
|
|
|
static void WriteTag(athena::io::YAMLDocWriter& cacheWriter,
|
|
const SObjectTag& pathTag, const hecl::ProjectPath& path)
|
|
{
|
|
char idStr[9];
|
|
snprintf(idStr, 9, "%08X", uint32_t(pathTag.id.Value()));
|
|
if (auto v = cacheWriter.enterSubVector(idStr))
|
|
{
|
|
cacheWriter.writeString(nullptr, pathTag.type.toString().c_str());
|
|
cacheWriter.writeString(nullptr, path.getAuxInfo().size() ?
|
|
(path.getRelativePathUTF8() + '|' + path.getAuxInfoUTF8()) :
|
|
path.getRelativePathUTF8());
|
|
}
|
|
}
|
|
|
|
static void WriteNameTag(athena::io::YAMLDocWriter& nameWriter,
|
|
const SObjectTag& pathTag,
|
|
const std::string& name)
|
|
{
|
|
char idStr[9];
|
|
snprintf(idStr, 9, "%08X", uint32_t(pathTag.id.Value()));
|
|
nameWriter.writeString(name.c_str(), idStr);
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::Clear()
|
|
{
|
|
m_tagToPath.clear();
|
|
m_pathToTag.clear();
|
|
m_catalogNameToTag.clear();
|
|
}
|
|
|
|
SObjectTag ProjectResourceFactoryBase::TagFromPath(const hecl::ProjectPath& path,
|
|
hecl::BlenderToken& btok) const
|
|
{
|
|
auto search = m_pathToTag.find(path.hash());
|
|
if (search != m_pathToTag.cend())
|
|
return search->second;
|
|
return BuildTagFromPath(path, btok);
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::ReadCatalog(const hecl::ProjectPath& catalogPath,
|
|
athena::io::YAMLDocWriter& nameWriter)
|
|
{
|
|
athena::io::FileReader freader(catalogPath.getAbsolutePath());
|
|
if (!freader.isOpen())
|
|
return;
|
|
|
|
athena::io::YAMLDocReader reader;
|
|
bool res = reader.parse(&freader);
|
|
if (!res)
|
|
return;
|
|
|
|
const athena::io::YAMLNode* root = reader.getRootNode();
|
|
for (const auto& p : root->m_mapChildren)
|
|
{
|
|
/* Hash as lowercase since lookup is case-insensitive */
|
|
std::string pLower = p.first;
|
|
std::transform(pLower.cbegin(), pLower.cend(), pLower.begin(), tolower);
|
|
|
|
/* Avoid redundant filesystem access for re-caches */
|
|
if (m_catalogNameToTag.find(pLower) != m_catalogNameToTag.cend())
|
|
continue;
|
|
|
|
athena::io::YAMLNode& node = *p.second;
|
|
hecl::ProjectPath path;
|
|
if (node.m_type == YAML_SCALAR_NODE)
|
|
{
|
|
path = hecl::ProjectPath(m_proj->getProjectWorkingPath(), node.m_scalarString);
|
|
}
|
|
else if (node.m_type == YAML_SEQUENCE_NODE)
|
|
{
|
|
if (node.m_seqChildren.size() >= 2)
|
|
path = hecl::ProjectPath(m_proj->getProjectWorkingPath(), node.m_seqChildren[0]->m_scalarString).
|
|
ensureAuxInfo(node.m_seqChildren[1]->m_scalarString);
|
|
else if (node.m_seqChildren.size() == 1)
|
|
path = hecl::ProjectPath(m_proj->getProjectWorkingPath(), node.m_seqChildren[0]->m_scalarString);
|
|
}
|
|
if (!path.isFileOrGlob())
|
|
continue;
|
|
SObjectTag pathTag = TagFromPath(path, m_backgroundBlender);
|
|
if (pathTag)
|
|
{
|
|
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
|
|
m_catalogNameToTag[pLower] = pathTag;
|
|
WriteNameTag(nameWriter, pathTag, p.first);
|
|
#if 0
|
|
fprintf(stderr, "%s %s %08X\n",
|
|
p.first.c_str(),
|
|
pathTag.type.toString().c_str(), uint32_t(pathTag.id));
|
|
#endif
|
|
}
|
|
}
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::BackgroundIndexRecursiveCatalogs(const hecl::ProjectPath& dir,
|
|
athena::io::YAMLDocWriter& nameWriter,
|
|
int level)
|
|
{
|
|
hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(),
|
|
hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted,
|
|
false, false, true);
|
|
|
|
/* Enumerate all items */
|
|
for (const hecl::DirectoryEnumerator::Entry& ent : dEnum)
|
|
{
|
|
hecl::ProjectPath path(dir, ent.m_name);
|
|
if (ent.m_isDir && level < 1)
|
|
BackgroundIndexRecursiveCatalogs(path, nameWriter, level+1);
|
|
else
|
|
{
|
|
if (!path.isFile())
|
|
continue;
|
|
|
|
/* Read catalog.yaml for .pak directory if exists */
|
|
if (level == 1 && !ent.m_name.compare(_S("!catalog.yaml")))
|
|
{
|
|
ReadCatalog(path, nameWriter);
|
|
continue;
|
|
}
|
|
}
|
|
|
|
/* bail if cancelled by client */
|
|
if (!m_backgroundRunning)
|
|
break;
|
|
}
|
|
}
|
|
|
|
#if DUMP_CACHE_FILL
|
|
static void DumpCacheAdd(const SObjectTag& pathTag, const hecl::ProjectPath& path)
|
|
{
|
|
fprintf(stderr, "%s %08X %s\n",
|
|
pathTag.type.toString().c_str(), uint32_t(pathTag.id.Value()),
|
|
path.getRelativePathUTF8().c_str());
|
|
}
|
|
#endif
|
|
|
|
bool ProjectResourceFactoryBase::AddFileToIndex(const hecl::ProjectPath& path,
|
|
athena::io::YAMLDocWriter& cacheWriter)
|
|
{
|
|
/* Avoid redundant filesystem access for re-caches */
|
|
if (m_pathToTag.find(path.hash()) != m_pathToTag.cend())
|
|
return true;
|
|
|
|
/* Try as glob */
|
|
hecl::ProjectPath asGlob = path.getWithExtension(_S(".*"), true);
|
|
if (m_pathToTag.find(asGlob.hash()) != m_pathToTag.cend())
|
|
return true;
|
|
|
|
/* Classify intermediate into tag */
|
|
SObjectTag pathTag = BuildTagFromPath(path, m_backgroundBlender);
|
|
if (pathTag)
|
|
{
|
|
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
|
|
bool useGlob = false;
|
|
|
|
/* Special multi-resource intermediates */
|
|
if (pathTag.type == SBIG('ANCS'))
|
|
{
|
|
hecl::BlenderConnection& conn = m_backgroundBlender.getBlenderConnection();
|
|
if (!conn.openBlend(path) || conn.getBlendType() != hecl::BlenderConnection::BlendType::Actor)
|
|
return false;
|
|
|
|
/* Transform tag to glob */
|
|
pathTag = {SBIG('ANCS'), asGlob.hash().val32()};
|
|
useGlob = true;
|
|
|
|
hecl::BlenderConnection::DataStream ds = conn.beginData();
|
|
std::vector<std::string> armatureNames = ds.getArmatureNames();
|
|
std::vector<std::string> subtypeNames = ds.getSubtypeNames();
|
|
std::vector<std::string> actionNames = ds.getActionNames();
|
|
|
|
for (const std::string& arm : armatureNames)
|
|
{
|
|
hecl::SystemStringView sysStr(arm);
|
|
hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".CINF"));
|
|
SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
|
|
m_tagToPath[pathTag] = subPath;
|
|
m_pathToTag[subPath.hash()] = pathTag;
|
|
WriteTag(cacheWriter, pathTag, subPath);
|
|
#if DUMP_CACHE_FILL
|
|
DumpCacheAdd(pathTag, subPath);
|
|
#endif
|
|
}
|
|
|
|
for (const std::string& sub : subtypeNames)
|
|
{
|
|
hecl::SystemStringView sysStr(sub);
|
|
hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".CSKR"));
|
|
SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
|
|
m_tagToPath[pathTag] = subPath;
|
|
m_pathToTag[subPath.hash()] = pathTag;
|
|
WriteTag(cacheWriter, pathTag, subPath);
|
|
#if DUMP_CACHE_FILL
|
|
DumpCacheAdd(pathTag, subPath);
|
|
#endif
|
|
}
|
|
|
|
for (const std::string& act : actionNames)
|
|
{
|
|
hecl::SystemStringView sysStr(act);
|
|
hecl::ProjectPath subPath = asGlob.ensureAuxInfo(sysStr.sys_str() + _S(".ANIM"));
|
|
SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
|
|
m_tagToPath[pathTag] = subPath;
|
|
m_pathToTag[subPath.hash()] = pathTag;
|
|
WriteTag(cacheWriter, pathTag, subPath);
|
|
#if DUMP_CACHE_FILL
|
|
DumpCacheAdd(pathTag, subPath);
|
|
#endif
|
|
}
|
|
}
|
|
else if (pathTag.type == SBIG('MLVL'))
|
|
{
|
|
/* Transform tag to glob */
|
|
pathTag = {SBIG('MLVL'), asGlob.hash().val32()};
|
|
useGlob = true;
|
|
|
|
hecl::ProjectPath subPath = asGlob.ensureAuxInfo(_S("MAPW"));
|
|
SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
|
|
m_tagToPath[pathTag] = subPath;
|
|
m_pathToTag[subPath.hash()] = pathTag;
|
|
WriteTag(cacheWriter, pathTag, subPath);
|
|
#if DUMP_CACHE_FILL
|
|
DumpCacheAdd(pathTag, subPath);
|
|
#endif
|
|
|
|
subPath = asGlob.ensureAuxInfo(_S("SAVW"));
|
|
pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
|
|
m_tagToPath[pathTag] = subPath;
|
|
m_pathToTag[subPath.hash()] = pathTag;
|
|
WriteTag(cacheWriter, pathTag, subPath);
|
|
#if DUMP_CACHE_FILL
|
|
DumpCacheAdd(pathTag, subPath);
|
|
#endif
|
|
}
|
|
else if (pathTag.type == SBIG('AGSC'))
|
|
{
|
|
/* Transform tag to glob */
|
|
pathTag = {SBIG('AGSC'), asGlob.hash().val32()};
|
|
useGlob = true;
|
|
}
|
|
else if (pathTag.type == SBIG('MREA'))
|
|
{
|
|
hecl::ProjectPath subPath = path.ensureAuxInfo(_S("PATH"));
|
|
SObjectTag pathTag = BuildTagFromPath(subPath, m_backgroundBlender);
|
|
m_tagToPath[pathTag] = subPath;
|
|
m_pathToTag[subPath.hash()] = pathTag;
|
|
WriteTag(cacheWriter, pathTag, subPath);
|
|
#if DUMP_CACHE_FILL
|
|
DumpCacheAdd(pathTag, subPath);
|
|
#endif
|
|
}
|
|
|
|
/* Cache in-memory */
|
|
const hecl::ProjectPath& usePath = useGlob ? asGlob : path;
|
|
m_tagToPath[pathTag] = usePath;
|
|
m_pathToTag[usePath.hash()] = pathTag;
|
|
WriteTag(cacheWriter, pathTag, usePath);
|
|
#if DUMP_CACHE_FILL
|
|
DumpCacheAdd(pathTag, usePath);
|
|
#endif
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::BackgroundIndexRecursiveProc(const hecl::ProjectPath& dir,
|
|
athena::io::YAMLDocWriter& cacheWriter,
|
|
athena::io::YAMLDocWriter& nameWriter,
|
|
int level)
|
|
{
|
|
hecl::DirectoryEnumerator dEnum(dir.getAbsolutePath(),
|
|
hecl::DirectoryEnumerator::Mode::DirsThenFilesSorted,
|
|
false, false, true);
|
|
|
|
/* Enumerate all items */
|
|
for (const hecl::DirectoryEnumerator::Entry& ent : dEnum)
|
|
{
|
|
hecl::ProjectPath path(dir, ent.m_name);
|
|
if (ent.m_isDir)
|
|
BackgroundIndexRecursiveProc(path, cacheWriter, nameWriter, level+1);
|
|
else
|
|
{
|
|
if (!path.isFile())
|
|
continue;
|
|
|
|
/* Read catalog.yaml for .pak directory if exists */
|
|
if (level == 1 && !ent.m_name.compare(_S("!catalog.yaml")))
|
|
{
|
|
ReadCatalog(path, nameWriter);
|
|
continue;
|
|
}
|
|
|
|
/* Index the regular file */
|
|
AddFileToIndex(path, cacheWriter);
|
|
}
|
|
|
|
/* bail if cancelled by client */
|
|
if (!m_backgroundRunning)
|
|
break;
|
|
}
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::BackgroundIndexProc()
|
|
{
|
|
logvisor::RegisterThreadName("Resource Index Thread");
|
|
|
|
hecl::ProjectPath tagCachePath(m_proj->getProjectCookedPath(*m_origSpec), _S("tag_cache.yaml"));
|
|
hecl::ProjectPath nameCachePath(m_proj->getProjectCookedPath(*m_origSpec), _S("name_cache.yaml"));
|
|
hecl::ProjectPath specRoot(m_proj->getProjectWorkingPath(), m_origSpec->m_name);
|
|
|
|
/* Cache will be overwritten with validated entries afterwards */
|
|
athena::io::YAMLDocWriter cacheWriter(nullptr);
|
|
athena::io::YAMLDocWriter nameWriter(nullptr);
|
|
|
|
/* Read in tag cache */
|
|
if (tagCachePath.isFile())
|
|
{
|
|
athena::io::FileReader reader(tagCachePath.getAbsolutePath());
|
|
if (reader.isOpen())
|
|
{
|
|
Log.report(logvisor::Info, _S("Cache index of '%s' loading"), m_origSpec->m_name);
|
|
athena::io::YAMLDocReader cacheReader;
|
|
if (cacheReader.parse(&reader))
|
|
{
|
|
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
|
|
size_t tagCount = cacheReader.getRootNode()->m_mapChildren.size();
|
|
m_tagToPath.reserve(tagCount);
|
|
m_pathToTag.reserve(tagCount);
|
|
size_t loadIdx = 0;
|
|
for (const auto& child : cacheReader.getRootNode()->m_mapChildren)
|
|
{
|
|
const athena::io::YAMLNode& node = *child.second;
|
|
unsigned long id = strtoul(child.first.c_str(), nullptr, 16);
|
|
hecl::FourCC type(node.m_seqChildren.at(0)->m_scalarString.c_str());
|
|
hecl::ProjectPath path(m_proj->getProjectWorkingPath(),
|
|
node.m_seqChildren.at(1)->m_scalarString);
|
|
|
|
if (path.isFileOrGlob())
|
|
{
|
|
SObjectTag pathTag(type, id);
|
|
m_tagToPath[pathTag] = path;
|
|
m_pathToTag[path.hash()] = pathTag;
|
|
WriteTag(cacheWriter, pathTag, path);
|
|
}
|
|
fprintf(stderr, "\r %" PRISize " / %" PRISize, ++loadIdx, tagCount);
|
|
}
|
|
fprintf(stderr, "\n");
|
|
}
|
|
Log.report(logvisor::Info, _S("Cache index of '%s' loaded; %d tags"),
|
|
m_origSpec->m_name, m_tagToPath.size());
|
|
|
|
if (nameCachePath.isFile())
|
|
{
|
|
/* Read in name cache */
|
|
Log.report(logvisor::Info, _S("Name index of '%s' loading"), m_origSpec->m_name);
|
|
athena::io::FileReader nreader(nameCachePath.getAbsolutePath());
|
|
athena::io::YAMLDocReader nameReader;
|
|
if (nameReader.parse(&nreader))
|
|
{
|
|
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
|
|
m_catalogNameToTag.reserve(nameReader.getRootNode()->m_mapChildren.size());
|
|
for (const auto& child : nameReader.getRootNode()->m_mapChildren)
|
|
{
|
|
unsigned long id = strtoul(child.second->m_scalarString.c_str(), nullptr, 16);
|
|
auto search = m_tagToPath.find(SObjectTag(FourCC(), uint32_t(id)));
|
|
if (search != m_tagToPath.cend())
|
|
{
|
|
std::string chLower = child.first;
|
|
std::transform(chLower.cbegin(), chLower.cend(), chLower.begin(), tolower);
|
|
m_catalogNameToTag[chLower] = search->first;
|
|
WriteNameTag(nameWriter, search->first, child.first);
|
|
}
|
|
}
|
|
}
|
|
Log.report(logvisor::Info, _S("Name index of '%s' loaded; %d names"),
|
|
m_origSpec->m_name, m_catalogNameToTag.size());
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Add special original IDs resource if exists (not name-cached to disk) */
|
|
hecl::ProjectPath oidsPath(specRoot, "!original_ids.yaml");
|
|
SObjectTag oidsTag = BuildTagFromPath(oidsPath, m_backgroundBlender);
|
|
if (oidsTag)
|
|
m_catalogNameToTag["mp1originalids"] = oidsTag;
|
|
|
|
Log.report(logvisor::Info, _S("Background index of '%s' started"), m_origSpec->m_name);
|
|
BackgroundIndexRecursiveProc(specRoot, cacheWriter, nameWriter, 0);
|
|
|
|
tagCachePath.makeDirChain(false);
|
|
athena::io::FileWriter twriter(tagCachePath.getAbsolutePath());
|
|
cacheWriter.finish(&twriter);
|
|
|
|
athena::io::FileWriter nwriter(nameCachePath.getAbsolutePath());
|
|
nameWriter.finish(&nwriter);
|
|
|
|
m_backgroundBlender.shutdown();
|
|
Log.report(logvisor::Info, _S("Background index of '%s' complete; %d tags, %d names"),
|
|
m_origSpec->m_name, m_tagToPath.size(), m_catalogNameToTag.size());
|
|
m_backgroundRunning = false;
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::CancelBackgroundIndex()
|
|
{
|
|
m_backgroundRunning = false;
|
|
if (m_backgroundIndexTh.joinable())
|
|
m_backgroundIndexTh.join();
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::BeginBackgroundIndex
|
|
(hecl::Database::Project& proj,
|
|
const hecl::Database::DataSpecEntry& origSpec,
|
|
const hecl::Database::DataSpecEntry& pcSpec)
|
|
{
|
|
CancelBackgroundIndex();
|
|
Clear();
|
|
m_proj = &proj;
|
|
m_origSpec = &origSpec;
|
|
m_pcSpec = &pcSpec;
|
|
m_cookSpec.reset(pcSpec.m_factory(proj, hecl::Database::DataSpecTool::Cook));
|
|
m_backgroundRunning = true;
|
|
m_backgroundIndexTh =
|
|
std::thread(std::bind(&ProjectResourceFactoryBase::BackgroundIndexProc, this));
|
|
}
|
|
|
|
hecl::ProjectPath ProjectResourceFactoryBase::GetCookedPath(const hecl::ProjectPath& working,
|
|
bool pcTarget) const
|
|
{
|
|
const hecl::Database::DataSpecEntry* spec = m_origSpec;
|
|
if (pcTarget)
|
|
spec = m_cookSpec->overrideDataSpec(working, m_pcSpec, hecl::SharedBlenderToken);
|
|
if (!spec)
|
|
return {};
|
|
return working.getCookedPath(*spec);
|
|
}
|
|
|
|
bool ProjectResourceFactoryBase::SyncCook(const hecl::ProjectPath& working)
|
|
{
|
|
Log.report(logvisor::Warning, _S("sync-cooking %s"), working.getRelativePath().c_str());
|
|
return m_clientProc.syncCook(working, m_cookSpec.get(), hecl::SharedBlenderToken);
|
|
}
|
|
|
|
CFactoryFnReturn ProjectResourceFactoryBase::BuildSync(const SObjectTag& tag,
|
|
const hecl::ProjectPath& path,
|
|
const CVParamTransfer& paramXfer,
|
|
CObjectReference* selfRef)
|
|
{
|
|
/* Ensure cooked rep is on the filesystem */
|
|
std::experimental::optional<athena::io::FileReader> fr;
|
|
if (!PrepForReadSync(tag, path, fr))
|
|
return {};
|
|
|
|
/* All good, build resource */
|
|
if (m_factoryMgr.CanMakeMemory(tag))
|
|
{
|
|
u32 length = fr->length();
|
|
std::unique_ptr<u8[]> memBuf = fr->readUBytes(length);
|
|
CFactoryFnReturn ret =
|
|
m_factoryMgr.MakeObjectFromMemory(tag, std::move(memBuf), length, false, paramXfer, selfRef);
|
|
Log.report(logvisor::Info, "sync-built %.4s %08X",
|
|
tag.type.toString().c_str(), u32(tag.id.Value()));
|
|
return ret;
|
|
}
|
|
|
|
CFactoryFnReturn ret = m_factoryMgr.MakeObject(tag, *fr, paramXfer, selfRef);
|
|
Log.report(logvisor::Info, "sync-built %.4s %08X",
|
|
tag.type.toString().c_str(), u32(tag.id.Value()));
|
|
return ret;
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::AsyncTask::EnsurePath(const urde::SObjectTag& tag,
|
|
const hecl::ProjectPath& path)
|
|
{
|
|
if (!m_workingPath)
|
|
{
|
|
m_workingPath = path;
|
|
|
|
/* Ensure requested resource is on the filesystem */
|
|
if (!path.isFileOrGlob())
|
|
{
|
|
Log.report(logvisor::Error, _S("unable to find resource path '%s'"),
|
|
path.getRelativePath().c_str());
|
|
m_failed = true;
|
|
return;
|
|
}
|
|
|
|
/* Cached resolve (try PC first, then original) */
|
|
m_cookedPath = path.getCookedPath(*m_parent.m_pcSpec);
|
|
if (!m_cookedPath.isFile())
|
|
m_cookedPath = path.getCookedPath(*m_parent.m_origSpec);
|
|
if (!m_cookedPath.isFile() ||
|
|
m_cookedPath.getModtime() < path.getModtime())
|
|
{
|
|
/* Last chance type validation */
|
|
urde::SObjectTag verifyTag = m_parent.TagFromPath(path, hecl::SharedBlenderToken);
|
|
if (verifyTag.type != tag.type)
|
|
{
|
|
Log.report(logvisor::Error, _S("%s: expected type '%.4s', found '%.4s'"),
|
|
path.getRelativePath().c_str(),
|
|
tag.type.toString().c_str(), verifyTag.type.toString().c_str());
|
|
m_failed = true;
|
|
return;
|
|
}
|
|
|
|
/* Get cooked representation path */
|
|
m_cookedPath = m_parent.GetCookedPath(path, true);
|
|
|
|
/* Perform mod-time comparison */
|
|
if (!m_cookedPath.isFile() ||
|
|
m_cookedPath.getModtime() < path.getModtime())
|
|
{
|
|
/* Start a background cook here */
|
|
m_cookTransaction = m_parent.m_clientProc.addCookTransaction(path, m_parent.m_cookSpec.get());
|
|
return;
|
|
}
|
|
}
|
|
|
|
CookComplete();
|
|
}
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::AsyncTask::CookComplete()
|
|
{
|
|
/* Ensure cooked rep is on the filesystem */
|
|
athena::io::FileReader fr(m_cookedPath.getAbsolutePath(), 32 * 1024, false);
|
|
if (fr.hasError())
|
|
{
|
|
Log.report(logvisor::Error, _S("unable to open cooked resource path '%s'"),
|
|
m_cookedPath.getAbsolutePath().c_str());
|
|
m_failed = true;
|
|
return;
|
|
}
|
|
|
|
/* Ready for buffer transaction at this point */
|
|
u32 availSz = std::max(0, s32(fr.length()) - s32(x14_resOffset));
|
|
x14_resSize = std::min(x14_resSize, availSz);
|
|
if (xc_targetDataRawPtr)
|
|
{
|
|
m_bufTransaction = m_parent.m_clientProc.addBufferTransaction(m_cookedPath,
|
|
xc_targetDataRawPtr,
|
|
x14_resSize, x14_resOffset);
|
|
}
|
|
else
|
|
{
|
|
x10_loadBuffer.reset(new u8[x14_resSize]);
|
|
m_bufTransaction = m_parent.m_clientProc.addBufferTransaction(m_cookedPath,
|
|
x10_loadBuffer.get(),
|
|
x14_resSize, x14_resOffset);
|
|
}
|
|
}
|
|
|
|
bool ProjectResourceFactoryBase::AsyncTask::AsyncPump()
|
|
{
|
|
if (m_failed)
|
|
return true;
|
|
|
|
if (m_bufTransaction)
|
|
{
|
|
if (m_bufTransaction->m_complete)
|
|
{
|
|
m_complete = true;
|
|
return true;
|
|
}
|
|
}
|
|
else if (m_cookTransaction)
|
|
{
|
|
if (m_cookTransaction->m_complete)
|
|
CookComplete();
|
|
}
|
|
|
|
return m_failed;
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::AsyncTask::WaitForComplete()
|
|
{
|
|
using ItType = std::unordered_map<SObjectTag, std::shared_ptr<AsyncTask>>::iterator;
|
|
ItType search = m_parent.m_asyncLoadList.find(x0_tag);
|
|
if (search == m_parent.m_asyncLoadList.end())
|
|
return;
|
|
for (ItType tmp = search ; !m_parent.AsyncPumpTask(tmp) ; tmp = search)
|
|
{std::this_thread::sleep_for(std::chrono::milliseconds(2));}
|
|
}
|
|
|
|
bool ProjectResourceFactoryBase::WaitForTagReady(const urde::SObjectTag& tag,
|
|
const hecl::ProjectPath*& pathOut)
|
|
{
|
|
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
|
|
auto search = m_tagToPath.find(tag);
|
|
if (search == m_tagToPath.end())
|
|
{
|
|
if (m_backgroundRunning)
|
|
{
|
|
while (m_backgroundRunning)
|
|
{
|
|
lk.unlock();
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(2));
|
|
lk.lock();
|
|
search = m_tagToPath.find(tag);
|
|
if (search != m_tagToPath.end())
|
|
break;
|
|
}
|
|
if (search == m_tagToPath.end())
|
|
return false;
|
|
}
|
|
else
|
|
return false;
|
|
}
|
|
lk.unlock();
|
|
pathOut = &search->second;
|
|
return true;
|
|
}
|
|
|
|
bool
|
|
ProjectResourceFactoryBase::PrepForReadSync(const SObjectTag& tag,
|
|
const hecl::ProjectPath& path,
|
|
std::experimental::optional<athena::io::FileReader>& fr)
|
|
{
|
|
/* Ensure requested resource is on the filesystem */
|
|
if (!path.isFileOrGlob())
|
|
{
|
|
Log.report(logvisor::Error, _S("unable to find resource path '%s'"),
|
|
path.getAbsolutePath().c_str());
|
|
return false;
|
|
}
|
|
|
|
/* Cached resolve (try PC first, then original) */
|
|
hecl::ProjectPath cooked = path.getCookedPath(*m_pcSpec);
|
|
if (!cooked.isFile())
|
|
cooked = path.getCookedPath(*m_origSpec);
|
|
if (!cooked.isFile() ||
|
|
cooked.getModtime() < path.getModtime())
|
|
{
|
|
/* Last chance type validation */
|
|
urde::SObjectTag verifyTag = TagFromPath(path, hecl::SharedBlenderToken);
|
|
if (verifyTag.type != tag.type)
|
|
{
|
|
Log.report(logvisor::Error, _S("%s: expected type '%.4s', found '%.4s'"),
|
|
path.getRelativePath().c_str(),
|
|
tag.type.toString().c_str(), verifyTag.type.toString().c_str());
|
|
return false;
|
|
}
|
|
|
|
/* Get cooked representation path */
|
|
cooked = GetCookedPath(path, true);
|
|
|
|
/* Perform mod-time comparison */
|
|
if (!cooked.isFile() ||
|
|
cooked.getModtime() < path.getModtime())
|
|
{
|
|
/* Do a blocking cook here */
|
|
if (!SyncCook(path))
|
|
{
|
|
Log.report(logvisor::Error, _S("unable to cook resource path '%s'"),
|
|
path.getAbsolutePath().c_str());
|
|
return false;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Ensure cooked rep is on the filesystem */
|
|
fr.emplace(cooked.getAbsolutePath(), 32 * 1024, false);
|
|
if (fr->hasError())
|
|
{
|
|
Log.report(logvisor::Error, _S("unable to open cooked resource path '%s'"),
|
|
cooked.getAbsolutePath().c_str());
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
std::unique_ptr<urde::IObj> ProjectResourceFactoryBase::Build(const urde::SObjectTag& tag,
|
|
const urde::CVParamTransfer& paramXfer,
|
|
CObjectReference* selfRef)
|
|
{
|
|
if (!tag.id.IsValid())
|
|
Log.report(logvisor::Fatal, "attempted to access null id on type '%s'", tag.type.toString().c_str());
|
|
|
|
const hecl::ProjectPath* resPath = nullptr;
|
|
if (!WaitForTagReady(tag, resPath))
|
|
return {};
|
|
auto asyncSearch = m_asyncLoadList.find(tag);
|
|
if (asyncSearch != m_asyncLoadList.end())
|
|
{
|
|
/* Async spinloop */
|
|
AsyncTask& task = *asyncSearch->second;
|
|
task.EnsurePath(task.x0_tag, *resPath);
|
|
|
|
/* Pump load pipeline (cooking if needed) */
|
|
while (!task.AsyncPump()) {std::this_thread::sleep_for(std::chrono::milliseconds(2));}
|
|
|
|
if (task.m_complete)
|
|
{
|
|
/* Load complete, build resource */
|
|
std::unique_ptr<IObj> newObj;
|
|
if (m_factoryMgr.CanMakeMemory(task.x0_tag))
|
|
{
|
|
newObj = m_factoryMgr.MakeObjectFromMemory(tag, std::move(task.x10_loadBuffer),
|
|
task.x14_resSize, false, task.x18_cvXfer, selfRef);
|
|
}
|
|
else
|
|
{
|
|
athena::io::MemoryReader mr(task.x10_loadBuffer.get(), task.x14_resSize);
|
|
newObj = m_factoryMgr.MakeObject(task.x0_tag, mr, task.x18_cvXfer, selfRef);
|
|
}
|
|
|
|
*task.xc_targetObjPtr = newObj.get();
|
|
Log.report(logvisor::Warning, "spin-built %.4s %08X",
|
|
task.x0_tag.type.toString().c_str(),
|
|
u32(task.x0_tag.id.Value()));
|
|
|
|
m_asyncLoadList.erase(asyncSearch);
|
|
return newObj;
|
|
}
|
|
Log.report(logvisor::Error, "unable to spin-build %.4s %08X",
|
|
task.x0_tag.type.toString().c_str(),
|
|
u32(task.x0_tag.id.Value()));
|
|
m_asyncLoadList.erase(asyncSearch);
|
|
return {};
|
|
}
|
|
|
|
/* Fall-back to sync build */
|
|
return BuildSync(tag, *resPath, paramXfer, selfRef);
|
|
}
|
|
|
|
std::shared_ptr<ProjectResourceFactoryBase::AsyncTask>
|
|
ProjectResourceFactoryBase::BuildAsyncInternal(const urde::SObjectTag& tag,
|
|
const urde::CVParamTransfer& paramXfer,
|
|
urde::IObj** objOut,
|
|
CObjectReference* selfRef)
|
|
{
|
|
if (m_asyncLoadList.find(tag) != m_asyncLoadList.end())
|
|
return {};
|
|
return m_asyncLoadList.emplace(std::make_pair(tag,
|
|
std::make_unique<AsyncTask>(*this, tag, objOut, paramXfer, selfRef))).first->second;
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::BuildAsync(const urde::SObjectTag& tag,
|
|
const urde::CVParamTransfer& paramXfer,
|
|
urde::IObj** objOut,
|
|
CObjectReference* selfRef)
|
|
{
|
|
if (!tag.id.IsValid())
|
|
Log.report(logvisor::Fatal, "attempted to access null id on type '%s'", tag.type.toString().c_str());
|
|
|
|
BuildAsyncInternal(tag, paramXfer, objOut, selfRef);
|
|
}
|
|
|
|
u32 ProjectResourceFactoryBase::ResourceSize(const SObjectTag& tag)
|
|
{
|
|
if (!tag.id.IsValid())
|
|
Log.report(logvisor::Fatal, "attempted to access null id on type '%s'", tag.type.toString().c_str());
|
|
|
|
/* Ensure resource at requested path is indexed and not cooking */
|
|
const hecl::ProjectPath* resPath = nullptr;
|
|
if (!WaitForTagReady(tag, resPath))
|
|
return {};
|
|
|
|
/* Ensure cooked rep is on the filesystem */
|
|
std::experimental::optional<athena::io::FileReader> fr;
|
|
if (!PrepForReadSync(tag, *resPath, fr))
|
|
return {};
|
|
|
|
return fr->length();
|
|
}
|
|
|
|
std::shared_ptr<ProjectResourceFactoryBase::AsyncTask>
|
|
ProjectResourceFactoryBase::LoadResourceAsync(const urde::SObjectTag& tag,
|
|
std::unique_ptr<u8[]>& target)
|
|
{
|
|
if (!tag.id.IsValid())
|
|
Log.report(logvisor::Fatal, "attempted to access null id");
|
|
if (m_asyncLoadList.find(tag) != m_asyncLoadList.end())
|
|
return {};
|
|
return m_asyncLoadList.emplace(std::make_pair(tag,
|
|
std::make_shared<AsyncTask>(*this, tag, target))).first->second;
|
|
}
|
|
|
|
std::shared_ptr<ProjectResourceFactoryBase::AsyncTask>
|
|
ProjectResourceFactoryBase::LoadResourcePartAsync(const urde::SObjectTag& tag,
|
|
u32 size, u32 off,
|
|
std::unique_ptr<u8[]>& target)
|
|
{
|
|
if (!tag.id.IsValid())
|
|
Log.report(logvisor::Fatal, "attempted to access null id");
|
|
if (m_asyncLoadList.find(tag) != m_asyncLoadList.end())
|
|
return {};
|
|
return m_asyncLoadList.emplace(std::make_pair(tag,
|
|
std::make_shared<AsyncTask>(*this, tag, target, size, off))).first->second;
|
|
}
|
|
|
|
std::shared_ptr<ProjectResourceFactoryBase::AsyncTask>
|
|
ProjectResourceFactoryBase::LoadResourcePartAsync(const urde::SObjectTag& tag, u32 size, u32 off, u8* target)
|
|
{
|
|
if (!tag.id.IsValid())
|
|
Log.report(logvisor::Fatal, "attempted to access null id");
|
|
if (m_asyncLoadList.find(tag) != m_asyncLoadList.end())
|
|
return {};
|
|
return m_asyncLoadList.emplace(std::make_pair(tag,
|
|
std::make_shared<AsyncTask>(*this, tag, target, size, off))).first->second;
|
|
}
|
|
|
|
std::unique_ptr<u8[]> ProjectResourceFactoryBase::LoadResourceSync(const urde::SObjectTag& tag)
|
|
{
|
|
if (!tag.id.IsValid())
|
|
Log.report(logvisor::Fatal, "attempted to access null id");
|
|
|
|
/* Ensure resource at requested path is indexed and not cooking */
|
|
const hecl::ProjectPath* resPath = nullptr;
|
|
if (!WaitForTagReady(tag, resPath))
|
|
return {};
|
|
|
|
/* Ensure cooked rep is on the filesystem */
|
|
std::experimental::optional<athena::io::FileReader> fr;
|
|
if (!PrepForReadSync(tag, *resPath, fr))
|
|
return {};
|
|
|
|
return fr->readUBytes(fr->length());
|
|
}
|
|
|
|
std::unique_ptr<u8[]> ProjectResourceFactoryBase::LoadResourcePartSync(const urde::SObjectTag& tag,
|
|
u32 size, u32 off)
|
|
{
|
|
if (!tag.id.IsValid())
|
|
Log.report(logvisor::Fatal, "attempted to access null id");
|
|
|
|
/* Ensure resource at requested path is indexed and not cooking */
|
|
const hecl::ProjectPath* resPath = nullptr;
|
|
if (!WaitForTagReady(tag, resPath))
|
|
return {};
|
|
|
|
/* Ensure cooked rep is on the filesystem */
|
|
std::experimental::optional<athena::io::FileReader> fr;
|
|
if (!PrepForReadSync(tag, *resPath, fr))
|
|
return {};
|
|
|
|
s32 sz = std::min(s32(size), std::max(0, s32(fr->length()) - s32(off)));
|
|
fr->seek(off, athena::SeekOrigin::Begin);
|
|
return fr->readUBytes(sz);
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::CancelBuild(const urde::SObjectTag& tag)
|
|
{
|
|
m_asyncLoadList.erase(tag);
|
|
}
|
|
|
|
bool ProjectResourceFactoryBase::CanBuild(const urde::SObjectTag& tag)
|
|
{
|
|
if (!tag.id.IsValid())
|
|
Log.report(logvisor::Fatal, "attempted to access null id");
|
|
|
|
const hecl::ProjectPath* resPath = nullptr;
|
|
if (!WaitForTagReady(tag, resPath))
|
|
return false;
|
|
|
|
if (resPath->isFile())
|
|
return true;
|
|
|
|
return false;
|
|
}
|
|
|
|
const urde::SObjectTag* ProjectResourceFactoryBase::GetResourceIdByName(const char* name) const
|
|
{
|
|
std::string lower = name;
|
|
std::transform(lower.cbegin(), lower.cend(), lower.begin(), tolower);
|
|
|
|
std::unique_lock<std::mutex> lk(const_cast<ProjectResourceFactoryBase*>(this)->m_backgroundIndexMutex);
|
|
auto search = m_catalogNameToTag.find(lower);
|
|
if (search == m_catalogNameToTag.end())
|
|
{
|
|
if (m_backgroundRunning)
|
|
{
|
|
while (m_backgroundRunning)
|
|
{
|
|
lk.unlock();
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(2));
|
|
lk.lock();
|
|
search = m_catalogNameToTag.find(lower);
|
|
if (search != m_catalogNameToTag.end())
|
|
break;
|
|
}
|
|
if (search == m_catalogNameToTag.end())
|
|
return nullptr;
|
|
}
|
|
else
|
|
return nullptr;
|
|
}
|
|
return &search->second;
|
|
}
|
|
|
|
FourCC ProjectResourceFactoryBase::GetResourceTypeById(CAssetId id) const
|
|
{
|
|
if (!id.IsValid())
|
|
return {};
|
|
|
|
std::unique_lock<std::mutex> lk(const_cast<ProjectResourceFactoryBase*>(this)->m_backgroundIndexMutex);
|
|
SObjectTag searchTag = {FourCC(), id};
|
|
auto search = m_tagToPath.find(searchTag);
|
|
if (search == m_tagToPath.end())
|
|
{
|
|
if (m_backgroundRunning)
|
|
{
|
|
while (m_backgroundRunning)
|
|
{
|
|
lk.unlock();
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(2));
|
|
lk.lock();
|
|
search = m_tagToPath.find(searchTag);
|
|
if (search != m_tagToPath.end())
|
|
break;
|
|
}
|
|
if (search == m_tagToPath.end())
|
|
return {};
|
|
}
|
|
else
|
|
return {};
|
|
}
|
|
|
|
return search->first.type;
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::EnumerateResources(const std::function<bool(const SObjectTag&)>& lambda) const
|
|
{
|
|
std::unique_lock<std::mutex> lk(const_cast<ProjectResourceFactoryBase*>(this)->m_backgroundIndexMutex);
|
|
while (m_backgroundRunning)
|
|
{
|
|
lk.unlock();
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(2));
|
|
lk.lock();
|
|
}
|
|
for (const auto& pair : m_tagToPath)
|
|
{
|
|
if (!lambda(pair.first))
|
|
break;
|
|
}
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::EnumerateNamedResources(
|
|
const std::function<bool(const std::string&, const SObjectTag&)>& lambda) const
|
|
{
|
|
std::unique_lock<std::mutex> lk(const_cast<ProjectResourceFactoryBase*>(this)->m_backgroundIndexMutex);
|
|
while (m_backgroundRunning)
|
|
{
|
|
lk.unlock();
|
|
std::this_thread::sleep_for(std::chrono::milliseconds(2));
|
|
lk.lock();
|
|
}
|
|
lk.unlock();
|
|
for (const auto& pair : m_catalogNameToTag)
|
|
{
|
|
if (!lambda(pair.first, pair.second))
|
|
break;
|
|
}
|
|
}
|
|
|
|
bool ProjectResourceFactoryBase::AsyncPumpTask(
|
|
std::unordered_map<SObjectTag, std::shared_ptr<AsyncTask>>::iterator& it)
|
|
{
|
|
/* Ensure requested resource is in the index */
|
|
std::unique_lock<std::mutex> lk(m_backgroundIndexMutex);
|
|
AsyncTask& task = *it->second;
|
|
auto search = m_tagToPath.find(task.x0_tag);
|
|
if (search == m_tagToPath.end())
|
|
{
|
|
if (!m_backgroundRunning)
|
|
{
|
|
Log.report(logvisor::Error, _S("unable to find async load resource (%s, %08X)"),
|
|
task.x0_tag.type.toString().c_str(), task.x0_tag.id);
|
|
it = m_asyncLoadList.erase(it);
|
|
}
|
|
return true;
|
|
}
|
|
lk.unlock();
|
|
task.EnsurePath(task.x0_tag, search->second);
|
|
|
|
/* Pump load pipeline (cooking if needed) */
|
|
if (task.AsyncPump())
|
|
{
|
|
if (task.m_complete)
|
|
{
|
|
/* Load complete, build resource */
|
|
if (task.xc_targetObjPtr)
|
|
{
|
|
/* Factory build */
|
|
std::unique_ptr<IObj> newObj;
|
|
if (m_factoryMgr.CanMakeMemory(task.x0_tag))
|
|
{
|
|
newObj = m_factoryMgr.MakeObjectFromMemory(task.x0_tag, std::move(task.x10_loadBuffer),
|
|
task.x14_resSize, false, task.x18_cvXfer,
|
|
task.m_selfRef);
|
|
}
|
|
else
|
|
{
|
|
athena::io::MemoryReader mr(task.x10_loadBuffer.get(), task.x14_resSize);
|
|
newObj = m_factoryMgr.MakeObject(task.x0_tag, mr, task.x18_cvXfer, task.m_selfRef);
|
|
}
|
|
|
|
*task.xc_targetObjPtr = newObj.release();
|
|
Log.report(logvisor::Info, "async-built %.4s %08X",
|
|
task.x0_tag.type.toString().c_str(),
|
|
u32(task.x0_tag.id.Value()));
|
|
}
|
|
else if (task.xc_targetDataPtr)
|
|
{
|
|
/* Buffer only */
|
|
*task.xc_targetDataPtr = std::move(task.x10_loadBuffer);
|
|
Log.report(logvisor::Info, "async-loaded %.4s %08X",
|
|
task.x0_tag.type.toString().c_str(),
|
|
u32(task.x0_tag.id.Value()));
|
|
}
|
|
else if (task.xc_targetDataRawPtr)
|
|
{
|
|
/* Buffer only raw */
|
|
Log.report(logvisor::Info, "async-loaded %.4s %08X",
|
|
task.x0_tag.type.toString().c_str(),
|
|
u32(task.x0_tag.id.Value()));
|
|
}
|
|
}
|
|
|
|
it = m_asyncLoadList.erase(it);
|
|
return true;
|
|
}
|
|
++it;
|
|
return false;
|
|
}
|
|
|
|
void ProjectResourceFactoryBase::AsyncIdle()
|
|
{
|
|
/* Consume completed transactions, they will be processed this cycle at the latest */
|
|
std::list<std::shared_ptr<hecl::ClientProcess::Transaction>> completed;
|
|
m_clientProc.swapCompletedQueue(completed);
|
|
|
|
/* Begin self-profiling loop */
|
|
std::chrono::steady_clock::time_point start = std::chrono::steady_clock::now();
|
|
for (auto it=m_asyncLoadList.begin() ; it != m_asyncLoadList.end() ;)
|
|
{
|
|
/* Allow 8 milliseconds (roughly 1/2 frame-time) for each async build cycle */
|
|
std::chrono::steady_clock::time_point resStart = std::chrono::steady_clock::now();
|
|
if (std::chrono::duration_cast<std::chrono::milliseconds>(resStart - start).count() > 8)
|
|
break;
|
|
|
|
AsyncPumpTask(it);
|
|
}
|
|
}
|
|
|
|
}
|