From 53256dea15ba37595b2958ed49249ddf370071ba Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Sun, 16 Aug 2015 13:00:33 -1000 Subject: [PATCH] proper recursive resource extraction --- DataSpec/DNACommon/ANCS.hpp | 4 +- DataSpec/DNACommon/CMDL.hpp | 21 +++++-- DataSpec/DNACommon/DNACommon.hpp | 91 +++++++++++++++++++++---------- DataSpec/DNAMP1/ANCS.hpp | 2 +- DataSpec/DNAMP1/CMDL.hpp | 2 +- DataSpec/DNAMP1/CMDLMaterials.hpp | 5 +- DataSpec/DNAMP2/ANCS.hpp | 2 +- DataSpec/DNAMP2/CMDL.hpp | 2 +- DataSpec/DNAMP2/CMDLMaterials.hpp | 5 +- DataSpec/DNAMP3/CMDL.hpp | 2 +- DataSpec/DNAMP3/CMDLMaterials.hpp | 5 +- 11 files changed, 95 insertions(+), 46 deletions(-) diff --git a/DataSpec/DNACommon/ANCS.hpp b/DataSpec/DNACommon/ANCS.hpp index 04d2b9e84..81c8bba27 100644 --- a/DataSpec/DNACommon/ANCS.hpp +++ b/DataSpec/DNACommon/ANCS.hpp @@ -26,7 +26,7 @@ bool ReadANCSToBlender(HECL::BlenderConnection& conn, const HECL::ProjectPath& outPath, PAKRouter& pakRouter, const typename PAKRouter::EntryType& entry, - const HECL::ProjectPath& masterShader, + const SpecBase& dataspec, bool force=false) { /* Extract character CMDL/CSKR first */ @@ -53,7 +53,7 @@ bool ReadANCSToBlender(HECL::BlenderConnection& conn, PAKEntryReadStream rs = cmdlE->beginReadStream(*node); DNACMDL::ReadCMDLToBlender - (conn, rs, pakRouter, *cmdlE, masterShader, &rigPair); + (conn, rs, pakRouter, *cmdlE, dataspec, &rigPair); conn.saveBlend(); } diff --git a/DataSpec/DNACommon/CMDL.hpp b/DataSpec/DNACommon/CMDL.hpp index cc14e6c92..056716419 100644 --- a/DataSpec/DNACommon/CMDL.hpp +++ b/DataSpec/DNACommon/CMDL.hpp @@ -4,6 +4,7 @@ #include "DNACommon.hpp" #include "BlenderConnection.hpp" #include "GX.hpp" +#include "TXTR.hpp" namespace Retro { @@ -116,13 +117,22 @@ void ReadMaterialSetToBlender_1_2(HECL::BlenderConnection::PyOutStream& os, const MaterialSet& matSet, const PAKRouter& pakRouter, const typename PAKRouter::EntryType& entry, - unsigned setIdx) + unsigned setIdx, + const SpecBase& dataspec) { /* Texmaps */ os << "texmap_list = []\n"; for (const UniqueID32& tex : matSet.head.textureIDs) { std::string texName = pakRouter.getBestEntryName(tex); + const NOD::DiscBase::IPartition::Node* node; + const typename PAKRouter::EntryType* texEntry = pakRouter.lookupEntry(tex, &node); + HECL::ProjectPath txtrPath = pakRouter.getWorking(texEntry); + if (txtrPath.getPathType() == HECL::ProjectPath::PT_NONE) + { + PAKEntryReadStream rs = texEntry->beginReadStream(*node); + TXTR::Extract(dataspec, rs, txtrPath); + } HECL::SystemString resPath = pakRouter.getResourceRelativePath(entry, tex); HECL::SystemUTF8View resPathView(resPath); os.format("if '%s' in bpy.data.textures:\n" @@ -151,7 +161,8 @@ void ReadMaterialSetToBlender_3(HECL::BlenderConnection::PyOutStream& os, const MaterialSet& matSet, const PAKRouter& pakRouter, const typename PAKRouter::EntryType& entry, - unsigned setIdx) + unsigned setIdx, + const SpecBase& dataspec) { unsigned m=0; for (const typename MaterialSet::Material& mat : matSet.materials) @@ -298,7 +309,7 @@ bool ReadCMDLToBlender(HECL::BlenderConnection& conn, Athena::io::IStreamReader& reader, PAKRouter& pakRouter, const typename PAKRouter::EntryType& entry, - const HECL::ProjectPath& masterShader, + const SpecBase& dataspec, const RIGPAIR* rp=nullptr) { Header head; @@ -407,7 +418,7 @@ bool ReadCMDLToBlender(HECL::BlenderConnection& conn, os.format("# Master shader library\n" "with bpy.data.libraries.load('%s', link=True, relative=True) as (data_from, data_to):\n" " data_to.node_groups = data_from.node_groups\n" - "\n", masterShader.getAbsolutePathUTF8().c_str()); + "\n", dataspec.getMasterShaderPath().getAbsolutePathUTF8().c_str()); MaterialSet::RegisterMaterialProps(os); os << "# Materials\n" @@ -508,7 +519,7 @@ bool ReadCMDLToBlender(HECL::BlenderConnection& conn, { MaterialSet matSet; matSet.read(reader); - matSet.readToBlender(os, pakRouter, entry, s); + matSet.readToBlender(os, pakRouter, entry, s, dataspec); if (!s) GetVertexAttributes(matSet, vertAttribs); } diff --git a/DataSpec/DNACommon/DNACommon.hpp b/DataSpec/DNACommon/DNACommon.hpp index 4e0c90d04..bc07ed7d8 100644 --- a/DataSpec/DNACommon/DNACommon.hpp +++ b/DataSpec/DNACommon/DNACommon.hpp @@ -336,16 +336,16 @@ public: private: const SpecBase& m_dataSpec; const std::vector* m_bridges = nullptr; + std::vector> m_bridgePaths; + size_t m_curBridgeIdx = 0; const HECL::ProjectPath& m_gameWorking; const HECL::ProjectPath& m_gameCooked; HECL::ProjectPath m_sharedWorking; HECL::ProjectPath m_sharedCooked; const PAKType* m_pak = nullptr; const NOD::DiscBase::IPartition::Node* m_node = nullptr; - HECL::ProjectPath m_pakWorking; - HECL::ProjectPath m_pakCooked; - std::unordered_map m_uniqueEntries; - std::unordered_map m_sharedEntries; + std::unordered_map> m_uniqueEntries; + std::unordered_map> m_sharedEntries; public: PAKRouter(const SpecBase& dataSpec, const HECL::ProjectPath& working, const HECL::ProjectPath& cooked) : m_dataSpec(dataSpec), @@ -354,46 +354,66 @@ public: void build(std::vector& bridges, std::function progress) { m_bridges = &bridges; + m_bridgePaths.clear(); + m_uniqueEntries.clear(); m_sharedEntries.clear(); size_t count = 0; float bridgesSz = bridges.size(); /* Route entries unique/shared per-pak */ + size_t bridgeIdx = 0; for (BRIDGETYPE& bridge : bridges) { + const std::string& name = bridge.getName(); + HECL::SystemStringView sysName(name); + + HECL::SystemString::const_iterator extit = sysName.sys_str().end() - 4; + HECL::SystemString baseName(sysName.sys_str().begin(), extit); + + m_bridgePaths.emplace_back(std::make_pair(HECL::ProjectPath(m_gameWorking, baseName), + HECL::ProjectPath(m_gameCooked, baseName))); + bridge.build(); const typename BRIDGETYPE::PAKType& pak = bridge.getPAK(); for (const auto& entry : pak.m_idMap) { - auto search = m_uniqueEntries.find(entry.first); - if (search != m_uniqueEntries.end()) + auto sSearch = m_sharedEntries.find(entry.first); + if (sSearch != m_sharedEntries.end()) + continue; + auto uSearch = m_uniqueEntries.find(entry.first); + if (uSearch != m_uniqueEntries.end()) { - m_uniqueEntries.erase(search); - m_sharedEntries.insert(entry); + m_uniqueEntries.erase(uSearch); + m_sharedEntries[entry.first] = std::make_pair(bridgeIdx, entry.second); } else - m_uniqueEntries.insert(entry); + m_uniqueEntries[entry.first] = std::make_pair(bridgeIdx, entry.second); } progress(++count / bridgesSz); + ++bridgeIdx; } } void enterPAKBridge(const BRIDGETYPE& pakBridge) - { - const std::string& name = pakBridge.getName(); - HECL::SystemStringView sysName(name); - - HECL::SystemString::const_iterator extit = sysName.sys_str().end() - 4; - HECL::SystemString baseName(sysName.sys_str().begin(), extit); - - m_pakWorking.assign(m_gameWorking, baseName); - m_pakWorking.makeDir(); - m_pakCooked.assign(m_gameCooked, baseName); - m_pakCooked.makeDir(); - - m_pak = &pakBridge.getPAK(); - m_node = &pakBridge.getNode(); + { + auto pit = m_bridgePaths.begin(); + size_t bridgeIdx = 0; + for (const BRIDGETYPE& bridge : *m_bridges) + { + if (&bridge == &pakBridge) + { + pit->first.makeDir(); + pit->second.makeDir(); + m_pak = &pakBridge.getPAK(); + m_node = &pakBridge.getNode(); + m_curBridgeIdx = bridgeIdx; + return; + } + ++pit; + ++bridgeIdx; + } + LogDNACommon.report(LogVisor::FatalError, "PAKBridge provided to PAKRouter::enterPAKBridge() was not part of build()"); } HECL::ProjectPath getWorking(const typename BRIDGETYPE::PAKType::Entry* entry, @@ -405,7 +425,9 @@ public: auto uniqueSearch = m_uniqueEntries.find(entry->id); if (uniqueSearch != m_uniqueEntries.end()) { - HECL::ProjectPath uniquePath = entry->unique.uniquePath(m_pakWorking); + const HECL::ProjectPath& pakPath = m_bridgePaths[uniqueSearch->second.first].first; + pakPath.makeDir(); + HECL::ProjectPath uniquePath = entry->unique.uniquePath(pakPath); HECL::SystemString entName = m_pak->bestEntryName(*entry); if (extractor.fileExts[0] && !extractor.fileExts[1]) entName += extractor.fileExts[0]; @@ -414,7 +436,8 @@ public: auto sharedSearch = m_sharedEntries.find(entry->id); if (sharedSearch != m_sharedEntries.end()) { - HECL::ProjectPath uniquePathPre = entry->unique.uniquePath(m_pakWorking); + const HECL::ProjectPath& pakPath = m_bridgePaths[m_curBridgeIdx].first; + HECL::ProjectPath uniquePathPre = entry->unique.uniquePath(pakPath); HECL::SystemString entBase = m_pak->bestEntryName(*entry); HECL::SystemString entName = entBase; if (extractor.fileExts[0] && !extractor.fileExts[1]) @@ -450,6 +473,11 @@ public: return getWorking(entry, BRIDGETYPE::LookupExtractor(*entry)); } + HECL::ProjectPath getWorking(const typename BRIDGETYPE::PAKType::IDType& id) const + { + return getWorking(lookupEntry(id)); + } + HECL::ProjectPath getCooked(const typename BRIDGETYPE::PAKType::Entry* entry) const { if (!m_pak) @@ -458,7 +486,9 @@ public: auto uniqueSearch = m_uniqueEntries.find(entry->id); if (uniqueSearch != m_uniqueEntries.end()) { - HECL::ProjectPath uniquePath = entry->unique.uniquePath(m_pakCooked); + const HECL::ProjectPath& pakPath = m_bridgePaths[uniqueSearch->second.first].second; + pakPath.makeDir(); + HECL::ProjectPath uniquePath = entry->unique.uniquePath(pakPath); return HECL::ProjectPath(uniquePath, m_pak->bestEntryName(*entry)); } auto sharedSearch = m_sharedEntries.find(entry->id); @@ -471,13 +501,18 @@ public: return HECL::ProjectPath(); } + HECL::ProjectPath getCooked(const typename BRIDGETYPE::PAKType::IDType& id) const + { + return getCooked(lookupEntry(id)); + } + HECL::SystemString getResourceRelativePath(const typename BRIDGETYPE::PAKType::Entry& a, const typename BRIDGETYPE::PAKType::IDType& b) const { if (!m_pak) LogDNACommon.report(LogVisor::FatalError, "PAKRouter::enterPAKBridge() must be called before PAKRouter::getResourceRelativePath()"); - const typename BRIDGETYPE::PAKType::Entry* be = m_pak->lookupEntry(b); + const typename BRIDGETYPE::PAKType::Entry* be = lookupEntry(b); if (!be) return HECL::SystemString(); HECL::ProjectPath aPath = getWorking(&a, BRIDGETYPE::LookupExtractor(a)); @@ -557,7 +592,7 @@ public: } const typename BRIDGETYPE::PAKType::Entry* lookupEntry(const typename BRIDGETYPE::PAKType::IDType& entry, - const NOD::DiscBase::IPartition::Node** nodeOut=nullptr) + const NOD::DiscBase::IPartition::Node** nodeOut=nullptr) const { if (!m_bridges) LogDNACommon.report(LogVisor::FatalError, diff --git a/DataSpec/DNAMP1/ANCS.hpp b/DataSpec/DNAMP1/ANCS.hpp index ea64d7d94..7a63b9fb8 100644 --- a/DataSpec/DNAMP1/ANCS.hpp +++ b/DataSpec/DNAMP1/ANCS.hpp @@ -401,7 +401,7 @@ struct ANCS : BigYAML { HECL::BlenderConnection& conn = HECL::BlenderConnection::SharedConnection(); DNAANCS::ReadANCSToBlender, ANCS, MaterialSet, 2> - (conn, ancs, blendPath, pakRouter, entry, dataSpec.getMasterShaderPath(), force); + (conn, ancs, blendPath, pakRouter, entry, dataSpec, force); conn.saveBlend(); } } diff --git a/DataSpec/DNAMP1/CMDL.hpp b/DataSpec/DNAMP1/CMDL.hpp index 66348412d..48910aa4d 100644 --- a/DataSpec/DNAMP1/CMDL.hpp +++ b/DataSpec/DNAMP1/CMDL.hpp @@ -26,7 +26,7 @@ struct CMDL if (!conn.createBlend(outPath.getAbsolutePath())) return false; DNACMDL::ReadCMDLToBlender, MaterialSet, std::pair, 2> - (conn, rs, pakRouter, entry, dataSpec.getMasterShaderPath()); + (conn, rs, pakRouter, entry, dataSpec); return conn.saveBlend(); } }; diff --git a/DataSpec/DNAMP1/CMDLMaterials.hpp b/DataSpec/DNAMP1/CMDLMaterials.hpp index 42ad94734..eeb935e09 100644 --- a/DataSpec/DNAMP1/CMDLMaterials.hpp +++ b/DataSpec/DNAMP1/CMDLMaterials.hpp @@ -309,9 +309,10 @@ struct MaterialSet : BigDNA inline void readToBlender(HECL::BlenderConnection::PyOutStream& os, const PAKRouter& pakRouter, const typename PAKRouter::EntryType& entry, - unsigned setIdx) + unsigned setIdx, + const SpecBase& dataspec) { - DNACMDL::ReadMaterialSetToBlender_1_2(os, *this, pakRouter, entry, setIdx); + DNACMDL::ReadMaterialSetToBlender_1_2(os, *this, pakRouter, entry, setIdx, dataspec); } }; diff --git a/DataSpec/DNAMP2/ANCS.hpp b/DataSpec/DNAMP2/ANCS.hpp index edfaca555..10ad88729 100644 --- a/DataSpec/DNAMP2/ANCS.hpp +++ b/DataSpec/DNAMP2/ANCS.hpp @@ -238,7 +238,7 @@ struct ANCS : BigYAML { HECL::BlenderConnection& conn = HECL::BlenderConnection::SharedConnection(); DNAANCS::ReadANCSToBlender, ANCS, MaterialSet, 4> - (conn, ancs, blendPath, pakRouter, entry, dataSpec.getMasterShaderPath(), force); + (conn, ancs, blendPath, pakRouter, entry, dataSpec, force); conn.saveBlend(); } } diff --git a/DataSpec/DNAMP2/CMDL.hpp b/DataSpec/DNAMP2/CMDL.hpp index 937a36778..3e3efd1a8 100644 --- a/DataSpec/DNAMP2/CMDL.hpp +++ b/DataSpec/DNAMP2/CMDL.hpp @@ -26,7 +26,7 @@ struct CMDL if (!conn.createBlend(outPath.getAbsolutePath())) return false; DNACMDL::ReadCMDLToBlender, MaterialSet, std::pair, 4> - (conn, rs, pakRouter, entry, dataSpec.getMasterShaderPath()); + (conn, rs, pakRouter, entry, dataSpec); return conn.saveBlend(); } }; diff --git a/DataSpec/DNAMP2/CMDLMaterials.hpp b/DataSpec/DNAMP2/CMDLMaterials.hpp index 84b565fec..9bd169ad1 100644 --- a/DataSpec/DNAMP2/CMDLMaterials.hpp +++ b/DataSpec/DNAMP2/CMDLMaterials.hpp @@ -70,9 +70,10 @@ struct MaterialSet : BigDNA inline void readToBlender(HECL::BlenderConnection::PyOutStream& os, const PAKRouter& pakRouter, const typename PAKRouter::EntryType& entry, - unsigned setIdx) + unsigned setIdx, + const SpecBase& dataspec) { - DNACMDL::ReadMaterialSetToBlender_1_2(os, *this, pakRouter, entry, setIdx); + DNACMDL::ReadMaterialSetToBlender_1_2(os, *this, pakRouter, entry, setIdx, dataspec); } }; diff --git a/DataSpec/DNAMP3/CMDL.hpp b/DataSpec/DNAMP3/CMDL.hpp index 9d857a2dc..e96d9ace1 100644 --- a/DataSpec/DNAMP3/CMDL.hpp +++ b/DataSpec/DNAMP3/CMDL.hpp @@ -26,7 +26,7 @@ struct CMDL if (!conn.createBlend(outPath.getAbsolutePath())) return false; DNACMDL::ReadCMDLToBlender, MaterialSet, std::pair, 5> - (conn, rs, pakRouter, entry, dataSpec.getMasterShaderPath()); + (conn, rs, pakRouter, entry, dataSpec); return conn.saveBlend(); } }; diff --git a/DataSpec/DNAMP3/CMDLMaterials.hpp b/DataSpec/DNAMP3/CMDLMaterials.hpp index 24913bebe..4a9d16f2b 100644 --- a/DataSpec/DNAMP3/CMDLMaterials.hpp +++ b/DataSpec/DNAMP3/CMDLMaterials.hpp @@ -191,9 +191,10 @@ struct MaterialSet : BigDNA inline void readToBlender(HECL::BlenderConnection::PyOutStream& os, const PAKRouter& pakRouter, const typename PAKRouter::EntryType& entry, - unsigned setIdx) + unsigned setIdx, + const SpecBase& dataspec) { - DNACMDL::ReadMaterialSetToBlender_3(os, *this, pakRouter, entry, setIdx); + DNACMDL::ReadMaterialSetToBlender_3(os, *this, pakRouter, entry, setIdx, dataspec); } };