AxioDL
/
metaforce
mirror of https://github.com/AxioDL/metaforce.git
2
0
Fork 0
Code Issues Projects Releases Wiki Activity

proper recursive resource extraction

This commit is contained in:
Jack Andersen 2015-08-16 13:00:33 -10:00
parent 0da7ab7e01
commit 53256dea15
11 changed files with 95 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
DataSpec/DNACommon/ANCS.hpp
View File

@ -26,7 +26,7 @@ bool ReadANCSToBlender(HECL::BlenderConnection& conn,
const HECL::ProjectPath& outPath, const HECL::ProjectPath& outPath,
PAKRouter& pakRouter, PAKRouter& pakRouter,
const typename PAKRouter::EntryType& entry, const typename PAKRouter::EntryType& entry,
const HECL::ProjectPath& masterShader, const SpecBase& dataspec,
bool force=false) bool force=false)
{ {
/* Extract character CMDL/CSKR first */ /* Extract character CMDL/CSKR first */
@ -53,7 +53,7 @@ bool ReadANCSToBlender(HECL::BlenderConnection& conn,
PAKEntryReadStream rs = cmdlE->beginReadStream(*node); PAKEntryReadStream rs = cmdlE->beginReadStream(*node);
DNACMDL::ReadCMDLToBlender<PAKRouter, MaterialSet, RIGPair, CMDLVersion> DNACMDL::ReadCMDLToBlender<PAKRouter, MaterialSet, RIGPair, CMDLVersion>
(conn, rs, pakRouter, *cmdlE, masterShader, &rigPair); (conn, rs, pakRouter, *cmdlE, dataspec, &rigPair);
conn.saveBlend(); conn.saveBlend();
} }

21
DataSpec/DNACommon/CMDL.hpp
View File

@ -4,6 +4,7 @@
#include "DNACommon.hpp" #include "DNACommon.hpp"
#include "BlenderConnection.hpp" #include "BlenderConnection.hpp"
#include "GX.hpp" #include "GX.hpp"
#include "TXTR.hpp"
namespace Retro namespace Retro
{ {
@ -116,13 +117,22 @@ void ReadMaterialSetToBlender_1_2(HECL::BlenderConnection::PyOutStream& os,
const MaterialSet& matSet, const MaterialSet& matSet,
const PAKRouter& pakRouter, const PAKRouter& pakRouter,
const typename PAKRouter::EntryType& entry, const typename PAKRouter::EntryType& entry,
unsigned setIdx) unsigned setIdx,
const SpecBase& dataspec)
{ {
/* Texmaps */ /* Texmaps */
os << "texmap_list = []\n"; os << "texmap_list = []\n";
for (const UniqueID32& tex : matSet.head.textureIDs) for (const UniqueID32& tex : matSet.head.textureIDs)
{ {
std::string texName = pakRouter.getBestEntryName(tex); std::string texName = pakRouter.getBestEntryName(tex);
const NOD::DiscBase::IPartition::Node* node;
const typename PAKRouter::EntryType* texEntry = pakRouter.lookupEntry(tex, &node);
HECL::ProjectPath txtrPath = pakRouter.getWorking(texEntry);
if (txtrPath.getPathType() == HECL::ProjectPath::PT_NONE)
{
PAKEntryReadStream rs = texEntry->beginReadStream(*node);
TXTR::Extract(dataspec, rs, txtrPath);
}
HECL::SystemString resPath = pakRouter.getResourceRelativePath(entry, tex); HECL::SystemString resPath = pakRouter.getResourceRelativePath(entry, tex);
HECL::SystemUTF8View resPathView(resPath); HECL::SystemUTF8View resPathView(resPath);
os.format("if '%s' in bpy.data.textures:\n" os.format("if '%s' in bpy.data.textures:\n"
@ -151,7 +161,8 @@ void ReadMaterialSetToBlender_3(HECL::BlenderConnection::PyOutStream& os,
const MaterialSet& matSet, const MaterialSet& matSet,
const PAKRouter& pakRouter, const PAKRouter& pakRouter,
const typename PAKRouter::EntryType& entry, const typename PAKRouter::EntryType& entry,
unsigned setIdx) unsigned setIdx,
const SpecBase& dataspec)
{ {
unsigned m=0; unsigned m=0;
for (const typename MaterialSet::Material& mat : matSet.materials) for (const typename MaterialSet::Material& mat : matSet.materials)
@ -298,7 +309,7 @@ bool ReadCMDLToBlender(HECL::BlenderConnection& conn,
Athena::io::IStreamReader& reader, Athena::io::IStreamReader& reader,
PAKRouter& pakRouter, PAKRouter& pakRouter,
const typename PAKRouter::EntryType& entry, const typename PAKRouter::EntryType& entry,
const HECL::ProjectPath& masterShader, const SpecBase& dataspec,
const RIGPAIR* rp=nullptr) const RIGPAIR* rp=nullptr)
{ {
Header head; Header head;
@ -407,7 +418,7 @@ bool ReadCMDLToBlender(HECL::BlenderConnection& conn,
os.format("# Master shader library\n" os.format("# Master shader library\n"
"with bpy.data.libraries.load('%s', link=True, relative=True) as (data_from, data_to):\n" "with bpy.data.libraries.load('%s', link=True, relative=True) as (data_from, data_to):\n"
" data_to.node_groups = data_from.node_groups\n" " data_to.node_groups = data_from.node_groups\n"
"\n", masterShader.getAbsolutePathUTF8().c_str()); "\n", dataspec.getMasterShaderPath().getAbsolutePathUTF8().c_str());
MaterialSet::RegisterMaterialProps(os); MaterialSet::RegisterMaterialProps(os);
os << "# Materials\n" os << "# Materials\n"
@ -508,7 +519,7 @@ bool ReadCMDLToBlender(HECL::BlenderConnection& conn,
{ {
MaterialSet matSet; MaterialSet matSet;
matSet.read(reader); matSet.read(reader);
matSet.readToBlender(os, pakRouter, entry, s); matSet.readToBlender(os, pakRouter, entry, s, dataspec);
if (!s) if (!s)
GetVertexAttributes(matSet, vertAttribs); GetVertexAttributes(matSet, vertAttribs);
} }

91
DataSpec/DNACommon/DNACommon.hpp
View File

@ -336,16 +336,16 @@ public:
private: private:
const SpecBase& m_dataSpec; const SpecBase& m_dataSpec;
const std::vector<BRIDGETYPE>* m_bridges = nullptr; const std::vector<BRIDGETYPE>* m_bridges = nullptr;
std::vector<std::pair<HECL::ProjectPath,HECL::ProjectPath>> m_bridgePaths;
size_t m_curBridgeIdx = 0;
const HECL::ProjectPath& m_gameWorking; const HECL::ProjectPath& m_gameWorking;
const HECL::ProjectPath& m_gameCooked; const HECL::ProjectPath& m_gameCooked;
HECL::ProjectPath m_sharedWorking; HECL::ProjectPath m_sharedWorking;
HECL::ProjectPath m_sharedCooked; HECL::ProjectPath m_sharedCooked;
const PAKType* m_pak = nullptr; const PAKType* m_pak = nullptr;
const NOD::DiscBase::IPartition::Node* m_node = nullptr; const NOD::DiscBase::IPartition::Node* m_node = nullptr;
HECL::ProjectPath m_pakWorking; std::unordered_map<IDType, std::pair<size_t, EntryType*>> m_uniqueEntries;
HECL::ProjectPath m_pakCooked; std::unordered_map<IDType, std::pair<size_t, EntryType*>> m_sharedEntries;
std::unordered_map<typename PAKType::IDType, typename PAKType::Entry*> m_uniqueEntries;
std::unordered_map<IDType, EntryType*> m_sharedEntries;
public: public:
PAKRouter(const SpecBase& dataSpec, const HECL::ProjectPath& working, const HECL::ProjectPath& cooked) PAKRouter(const SpecBase& dataSpec, const HECL::ProjectPath& working, const HECL::ProjectPath& cooked)
: m_dataSpec(dataSpec), : m_dataSpec(dataSpec),
@ -354,46 +354,66 @@ public:
void build(std::vector<BRIDGETYPE>& bridges, std::function<void(float)> progress) void build(std::vector<BRIDGETYPE>& bridges, std::function<void(float)> progress)
{ {
m_bridges = &bridges; m_bridges = &bridges;
m_bridgePaths.clear();
m_uniqueEntries.clear(); m_uniqueEntries.clear();
m_sharedEntries.clear(); m_sharedEntries.clear();
size_t count = 0; size_t count = 0;
float bridgesSz = bridges.size(); float bridgesSz = bridges.size();
/* Route entries unique/shared per-pak */ /* Route entries unique/shared per-pak */
size_t bridgeIdx = 0;
for (BRIDGETYPE& bridge : bridges) for (BRIDGETYPE& bridge : bridges)
{ {
const std::string& name = bridge.getName();
HECL::SystemStringView sysName(name);
HECL::SystemString::const_iterator extit = sysName.sys_str().end() - 4;
HECL::SystemString baseName(sysName.sys_str().begin(), extit);
m_bridgePaths.emplace_back(std::make_pair(HECL::ProjectPath(m_gameWorking, baseName),
HECL::ProjectPath(m_gameCooked, baseName)));
bridge.build(); bridge.build();
const typename BRIDGETYPE::PAKType& pak = bridge.getPAK(); const typename BRIDGETYPE::PAKType& pak = bridge.getPAK();
for (const auto& entry : pak.m_idMap) for (const auto& entry : pak.m_idMap)
{ {
auto search = m_uniqueEntries.find(entry.first); auto sSearch = m_sharedEntries.find(entry.first);
if (search != m_uniqueEntries.end()) if (sSearch != m_sharedEntries.end())
continue;
auto uSearch = m_uniqueEntries.find(entry.first);
if (uSearch != m_uniqueEntries.end())
{ {
m_uniqueEntries.erase(search); m_uniqueEntries.erase(uSearch);
m_sharedEntries.insert(entry); m_sharedEntries[entry.first] = std::make_pair(bridgeIdx, entry.second);
} }
else else
m_uniqueEntries.insert(entry); m_uniqueEntries[entry.first] = std::make_pair(bridgeIdx, entry.second);
} }
progress(++count / bridgesSz); progress(++count / bridgesSz);
++bridgeIdx;
} }
} }
void enterPAKBridge(const BRIDGETYPE& pakBridge) void enterPAKBridge(const BRIDGETYPE& pakBridge)
{ {
const std::string& name = pakBridge.getName(); auto pit = m_bridgePaths.begin();
HECL::SystemStringView sysName(name); size_t bridgeIdx = 0;
for (const BRIDGETYPE& bridge : *m_bridges)
HECL::SystemString::const_iterator extit = sysName.sys_str().end() - 4; {
HECL::SystemString baseName(sysName.sys_str().begin(), extit); if (&bridge == &pakBridge)
{
m_pakWorking.assign(m_gameWorking, baseName); pit->first.makeDir();
m_pakWorking.makeDir(); pit->second.makeDir();
m_pakCooked.assign(m_gameCooked, baseName); m_pak = &pakBridge.getPAK();
m_pakCooked.makeDir(); m_node = &pakBridge.getNode();
m_curBridgeIdx = bridgeIdx;
m_pak = &pakBridge.getPAK(); return;
m_node = &pakBridge.getNode(); }
++pit;
++bridgeIdx;
}
LogDNACommon.report(LogVisor::FatalError, "PAKBridge provided to PAKRouter::enterPAKBridge() was not part of build()");
} }
HECL::ProjectPath getWorking(const typename BRIDGETYPE::PAKType::Entry* entry, HECL::ProjectPath getWorking(const typename BRIDGETYPE::PAKType::Entry* entry,
@ -405,7 +425,9 @@ public:
auto uniqueSearch = m_uniqueEntries.find(entry->id); auto uniqueSearch = m_uniqueEntries.find(entry->id);
if (uniqueSearch != m_uniqueEntries.end()) if (uniqueSearch != m_uniqueEntries.end())
{ {
HECL::ProjectPath uniquePath = entry->unique.uniquePath(m_pakWorking); const HECL::ProjectPath& pakPath = m_bridgePaths[uniqueSearch->second.first].first;
pakPath.makeDir();
HECL::ProjectPath uniquePath = entry->unique.uniquePath(pakPath);
HECL::SystemString entName = m_pak->bestEntryName(*entry); HECL::SystemString entName = m_pak->bestEntryName(*entry);
if (extractor.fileExts[0] && !extractor.fileExts[1]) if (extractor.fileExts[0] && !extractor.fileExts[1])
entName += extractor.fileExts[0]; entName += extractor.fileExts[0];
@ -414,7 +436,8 @@ public:
auto sharedSearch = m_sharedEntries.find(entry->id); auto sharedSearch = m_sharedEntries.find(entry->id);
if (sharedSearch != m_sharedEntries.end()) if (sharedSearch != m_sharedEntries.end())
{ {
HECL::ProjectPath uniquePathPre = entry->unique.uniquePath(m_pakWorking); const HECL::ProjectPath& pakPath = m_bridgePaths[m_curBridgeIdx].first;
HECL::ProjectPath uniquePathPre = entry->unique.uniquePath(pakPath);
HECL::SystemString entBase = m_pak->bestEntryName(*entry); HECL::SystemString entBase = m_pak->bestEntryName(*entry);
HECL::SystemString entName = entBase; HECL::SystemString entName = entBase;
if (extractor.fileExts[0] && !extractor.fileExts[1]) if (extractor.fileExts[0] && !extractor.fileExts[1])
@ -450,6 +473,11 @@ public:
return getWorking(entry, BRIDGETYPE::LookupExtractor(*entry)); return getWorking(entry, BRIDGETYPE::LookupExtractor(*entry));
} }
HECL::ProjectPath getWorking(const typename BRIDGETYPE::PAKType::IDType& id) const
{
return getWorking(lookupEntry(id));
}
HECL::ProjectPath getCooked(const typename BRIDGETYPE::PAKType::Entry* entry) const HECL::ProjectPath getCooked(const typename BRIDGETYPE::PAKType::Entry* entry) const
{ {
if (!m_pak) if (!m_pak)
@ -458,7 +486,9 @@ public:
auto uniqueSearch = m_uniqueEntries.find(entry->id); auto uniqueSearch = m_uniqueEntries.find(entry->id);
if (uniqueSearch != m_uniqueEntries.end()) if (uniqueSearch != m_uniqueEntries.end())
{ {
HECL::ProjectPath uniquePath = entry->unique.uniquePath(m_pakCooked); const HECL::ProjectPath& pakPath = m_bridgePaths[uniqueSearch->second.first].second;
pakPath.makeDir();
HECL::ProjectPath uniquePath = entry->unique.uniquePath(pakPath);
return HECL::ProjectPath(uniquePath, m_pak->bestEntryName(*entry)); return HECL::ProjectPath(uniquePath, m_pak->bestEntryName(*entry));
} }
auto sharedSearch = m_sharedEntries.find(entry->id); auto sharedSearch = m_sharedEntries.find(entry->id);
@ -471,13 +501,18 @@ public:
return HECL::ProjectPath(); return HECL::ProjectPath();
} }
HECL::ProjectPath getCooked(const typename BRIDGETYPE::PAKType::IDType& id) const
{
return getCooked(lookupEntry(id));
}
HECL::SystemString getResourceRelativePath(const typename BRIDGETYPE::PAKType::Entry& a, HECL::SystemString getResourceRelativePath(const typename BRIDGETYPE::PAKType::Entry& a,
const typename BRIDGETYPE::PAKType::IDType& b) const const typename BRIDGETYPE::PAKType::IDType& b) const
{ {
if (!m_pak) if (!m_pak)
LogDNACommon.report(LogVisor::FatalError, LogDNACommon.report(LogVisor::FatalError,
"PAKRouter::enterPAKBridge() must be called before PAKRouter::getResourceRelativePath()"); "PAKRouter::enterPAKBridge() must be called before PAKRouter::getResourceRelativePath()");
const typename BRIDGETYPE::PAKType::Entry* be = m_pak->lookupEntry(b); const typename BRIDGETYPE::PAKType::Entry* be = lookupEntry(b);
if (!be) if (!be)
return HECL::SystemString(); return HECL::SystemString();
HECL::ProjectPath aPath = getWorking(&a, BRIDGETYPE::LookupExtractor(a)); HECL::ProjectPath aPath = getWorking(&a, BRIDGETYPE::LookupExtractor(a));
@ -557,7 +592,7 @@ public:
} }
const typename BRIDGETYPE::PAKType::Entry* lookupEntry(const typename BRIDGETYPE::PAKType::IDType& entry, const typename BRIDGETYPE::PAKType::Entry* lookupEntry(const typename BRIDGETYPE::PAKType::IDType& entry,
const NOD::DiscBase::IPartition::Node** nodeOut=nullptr) const NOD::DiscBase::IPartition::Node** nodeOut=nullptr) const
{ {
if (!m_bridges) if (!m_bridges)
LogDNACommon.report(LogVisor::FatalError, LogDNACommon.report(LogVisor::FatalError,

2
DataSpec/DNAMP1/ANCS.hpp
View File

@ -401,7 +401,7 @@ struct ANCS : BigYAML
{ {
HECL::BlenderConnection& conn = HECL::BlenderConnection::SharedConnection(); HECL::BlenderConnection& conn = HECL::BlenderConnection::SharedConnection();
DNAANCS::ReadANCSToBlender<PAKRouter<PAKBridge>, ANCS, MaterialSet, 2> DNAANCS::ReadANCSToBlender<PAKRouter<PAKBridge>, ANCS, MaterialSet, 2>
(conn, ancs, blendPath, pakRouter, entry, dataSpec.getMasterShaderPath(), force); (conn, ancs, blendPath, pakRouter, entry, dataSpec, force);
conn.saveBlend(); conn.saveBlend();
} }
} }

2
DataSpec/DNAMP1/CMDL.hpp
View File

@ -26,7 +26,7 @@ struct CMDL
if (!conn.createBlend(outPath.getAbsolutePath())) if (!conn.createBlend(outPath.getAbsolutePath()))
return false; return false;
DNACMDL::ReadCMDLToBlender<PAKRouter<PAKBridge>, MaterialSet, std::pair<CSKR*,CINF*>, 2> DNACMDL::ReadCMDLToBlender<PAKRouter<PAKBridge>, MaterialSet, std::pair<CSKR*,CINF*>, 2>
(conn, rs, pakRouter, entry, dataSpec.getMasterShaderPath()); (conn, rs, pakRouter, entry, dataSpec);
return conn.saveBlend(); return conn.saveBlend();
} }
}; };

5
DataSpec/DNAMP1/CMDLMaterials.hpp
View File

@ -309,9 +309,10 @@ struct MaterialSet : BigDNA
inline void readToBlender(HECL::BlenderConnection::PyOutStream& os, inline void readToBlender(HECL::BlenderConnection::PyOutStream& os,
const PAKRouter<PAKBridge>& pakRouter, const PAKRouter<PAKBridge>& pakRouter,
const typename PAKRouter<PAKBridge>::EntryType& entry, const typename PAKRouter<PAKBridge>::EntryType& entry,
unsigned setIdx) unsigned setIdx,
const SpecBase& dataspec)
{ {
DNACMDL::ReadMaterialSetToBlender_1_2(os, *this, pakRouter, entry, setIdx); DNACMDL::ReadMaterialSetToBlender_1_2(os, *this, pakRouter, entry, setIdx, dataspec);
} }
}; };

2
DataSpec/DNAMP2/ANCS.hpp
View File

@ -238,7 +238,7 @@ struct ANCS : BigYAML
{ {
HECL::BlenderConnection& conn = HECL::BlenderConnection::SharedConnection(); HECL::BlenderConnection& conn = HECL::BlenderConnection::SharedConnection();
DNAANCS::ReadANCSToBlender<PAKRouter<PAKBridge>, ANCS, MaterialSet, 4> DNAANCS::ReadANCSToBlender<PAKRouter<PAKBridge>, ANCS, MaterialSet, 4>
(conn, ancs, blendPath, pakRouter, entry, dataSpec.getMasterShaderPath(), force); (conn, ancs, blendPath, pakRouter, entry, dataSpec, force);
conn.saveBlend(); conn.saveBlend();
} }
} }

2
DataSpec/DNAMP2/CMDL.hpp
View File

@ -26,7 +26,7 @@ struct CMDL
if (!conn.createBlend(outPath.getAbsolutePath())) if (!conn.createBlend(outPath.getAbsolutePath()))
return false; return false;
DNACMDL::ReadCMDLToBlender<PAKRouter<PAKBridge>, MaterialSet, std::pair<CSKR*,CINF*>, 4> DNACMDL::ReadCMDLToBlender<PAKRouter<PAKBridge>, MaterialSet, std::pair<CSKR*,CINF*>, 4>
(conn, rs, pakRouter, entry, dataSpec.getMasterShaderPath()); (conn, rs, pakRouter, entry, dataSpec);
return conn.saveBlend(); return conn.saveBlend();
} }
}; };

5
DataSpec/DNAMP2/CMDLMaterials.hpp
View File

@ -70,9 +70,10 @@ struct MaterialSet : BigDNA
inline void readToBlender(HECL::BlenderConnection::PyOutStream& os, inline void readToBlender(HECL::BlenderConnection::PyOutStream& os,
const PAKRouter<PAKBridge>& pakRouter, const PAKRouter<PAKBridge>& pakRouter,
const typename PAKRouter<PAKBridge>::EntryType& entry, const typename PAKRouter<PAKBridge>::EntryType& entry,
unsigned setIdx) unsigned setIdx,
const SpecBase& dataspec)
{ {
DNACMDL::ReadMaterialSetToBlender_1_2(os, *this, pakRouter, entry, setIdx); DNACMDL::ReadMaterialSetToBlender_1_2(os, *this, pakRouter, entry, setIdx, dataspec);
} }
}; };

2
DataSpec/DNAMP3/CMDL.hpp
View File

@ -26,7 +26,7 @@ struct CMDL
if (!conn.createBlend(outPath.getAbsolutePath())) if (!conn.createBlend(outPath.getAbsolutePath()))
return false; return false;
DNACMDL::ReadCMDLToBlender<PAKRouter<PAKBridge>, MaterialSet, std::pair<CSKR*,CINF*>, 5> DNACMDL::ReadCMDLToBlender<PAKRouter<PAKBridge>, MaterialSet, std::pair<CSKR*,CINF*>, 5>
(conn, rs, pakRouter, entry, dataSpec.getMasterShaderPath()); (conn, rs, pakRouter, entry, dataSpec);
return conn.saveBlend(); return conn.saveBlend();
} }
}; };

5
DataSpec/DNAMP3/CMDLMaterials.hpp
View File

@ -191,9 +191,10 @@ struct MaterialSet : BigDNA
inline void readToBlender(HECL::BlenderConnection::PyOutStream& os, inline void readToBlender(HECL::BlenderConnection::PyOutStream& os,
const PAKRouter<PAKBridge>& pakRouter, const PAKRouter<PAKBridge>& pakRouter,
const typename PAKRouter<PAKBridge>::EntryType& entry, const typename PAKRouter<PAKBridge>::EntryType& entry,
unsigned setIdx) unsigned setIdx,
const SpecBase& dataspec)
{ {
DNACMDL::ReadMaterialSetToBlender_3(os, *this, pakRouter, entry, setIdx); DNACMDL::ReadMaterialSetToBlender_3(os, *this, pakRouter, entry, setIdx, dataspec);
} }
}; };