#ifndef __DNA_COMMON_HPP__ #define __DNA_COMMON_HPP__ #include #include #include #include "HECL/HECL.hpp" #include "HECL/Database.hpp" #include "../SpecBase.hpp" namespace Retro { extern LogVisor::LogModule LogDNACommon; /* This comes up a great deal */ typedef Athena::io::DNA BigDNA; typedef Athena::io::DNAYaml BigYAML; /* FourCC with DNA read/write */ class FourCC final : public BigYAML, public HECL::FourCC { public: FourCC() : HECL::FourCC() {} FourCC(const HECL::FourCC& other) : HECL::FourCC() {num = other.toUint32();} FourCC(const char* name) : HECL::FourCC(name) {} FourCC(uint32_t n) : HECL::FourCC(n) {} Delete expl; inline void read(Athena::io::IStreamReader& reader) {reader.readUBytesToBuf(fcc, 4);} inline void write(Athena::io::IStreamWriter& writer) const {writer.writeUBytes((atUint8*)fcc, 4);} inline void fromYAML(Athena::io::YAMLDocReader& reader) {std::string rs = reader.readString(nullptr); strncpy(fcc, rs.c_str(), 4);} inline void toYAML(Athena::io::YAMLDocWriter& writer) const {writer.writeString(nullptr, std::string(fcc, 4));} }; /* PAK 32-bit Unique ID */ class UniqueID32 : public BigYAML { uint32_t m_id = 0xffffffff; public: Delete expl; inline operator bool() const {return m_id != 0xffffffff;} inline void read(Athena::io::IStreamReader& reader) {m_id = reader.readUint32Big();} inline void write(Athena::io::IStreamWriter& writer) const {writer.writeUint32Big(m_id);} inline void fromYAML(Athena::io::YAMLDocReader& reader) {m_id = reader.readUint32(nullptr);} inline void toYAML(Athena::io::YAMLDocWriter& writer) const {writer.writeUint32(nullptr, m_id);} inline bool operator!=(const UniqueID32& other) const {return m_id != other.m_id;} inline bool operator==(const UniqueID32& other) const {return m_id == other.m_id;} inline uint32_t toUint32() const {return m_id;} inline std::string toString() const { char buf[9]; snprintf(buf, 9, "%08X", m_id); return std::string(buf); } }; /* PAK 64-bit Unique ID */ class UniqueID64 : public BigDNA { uint64_t m_id = 0xffffffffffffffff; public: Delete expl; inline operator bool() const {return m_id != 0xffffffffffffffff;} inline void read(Athena::io::IStreamReader& reader) {m_id = reader.readUint64Big();} inline void write(Athena::io::IStreamWriter& writer) const {writer.writeUint64Big(m_id);} inline bool operator!=(const UniqueID64& other) const {return m_id != other.m_id;} inline bool operator==(const UniqueID64& other) const {return m_id == other.m_id;} inline uint64_t toUint64() const {return m_id;} inline std::string toString() const { char buf[17]; snprintf(buf, 17, "%016lX", m_id); return std::string(buf); } }; /* PAK 128-bit Unique ID */ class UniqueID128 : public BigDNA { union { uint64_t m_id[2]; #if __SSE__ __m128i m_id128; #endif }; public: Delete expl; UniqueID128() {m_id[0]=0xffffffffffffffff; m_id[1]=0xffffffffffffffff;} inline operator bool() const {return m_id[0] != 0xffffffffffffffff && m_id[1] != 0xffffffffffffffff;} inline void read(Athena::io::IStreamReader& reader) { m_id[0] = reader.readUint64Big(); m_id[1] = reader.readUint64Big(); } inline void write(Athena::io::IStreamWriter& writer) const { writer.writeUint64Big(m_id[0]); writer.writeUint64Big(m_id[1]); } inline bool operator!=(const UniqueID128& other) const { #if __SSE__ __m128i vcmp = _mm_cmpeq_epi32(m_id128, other.m_id128); int vmask = _mm_movemask_epi8(vcmp); return vmask != 0xffff; #else return (m_id[0] != other.m_id[0]) || (m_id[1] != other.m_id[1]); #endif } inline bool operator==(const UniqueID128& other) const { #if __SSE__ __m128i vcmp = _mm_cmpeq_epi32(m_id128, other.m_id128); int vmask = _mm_movemask_epi8(vcmp); return vmask == 0xffff; #else return (m_id[0] == other.m_id[0]) && (m_id[1] == other.m_id[1]); #endif } inline uint64_t toHighUint64() const {return m_id[0];} inline uint64_t toLowUint64() const {return m_id[1];} inline std::string toString() const { char buf[33]; snprintf(buf, 33, "%016lX%016lX", m_id[0], m_id[1]); return std::string(buf); } }; /* Case-insensitive comparator for std::map sorting */ struct CaseInsensitiveCompare { inline bool operator()(const std::string& lhs, const std::string& rhs) const { #if _WIN32 if (_stricmp(lhs.c_str(), rhs.c_str()) < 0) #else if (strcasecmp(lhs.c_str(), rhs.c_str()) < 0) #endif return true; return false; } #if _WIN32 inline bool operator()(const std::wstring& lhs, const std::wstring& rhs) const { if (_wcsicmp(lhs.c_str(), rhs.c_str()) < 0) return true; return false; } #endif }; /* Word Bitmap reader/writer */ struct WordBitmap { std::vector m_words; size_t m_bitCount = 0; void read(Athena::io::IStreamReader& reader, size_t bitCount) { m_bitCount = bitCount; size_t wordCount = (bitCount + 31) / 32; m_words.clear(); m_words.reserve(wordCount); for (size_t w=0 ; w= m_words.size()) return false; size_t wordCur = idx % 32; return (m_words[wordIdx] >> wordCur) & 0x1; } void setBit(size_t idx) { size_t wordIdx = idx / 32; while (wordIdx >= m_words.size()) m_words.push_back(0); size_t wordCur = idx % 32; m_words[wordIdx] |= (1 << wordCur); } void unsetBit(size_t idx) { size_t wordIdx = idx / 32; while (wordIdx >= m_words.size()) m_words.push_back(0); size_t wordCur = idx % 32; m_words[wordIdx] &= ~(1 << wordCur); } void clear() { m_words.clear(); } class Iterator : public std::iterator { friend class WordBitmap; const WordBitmap& m_bmp; size_t m_idx = 0; Iterator(const WordBitmap& bmp, size_t idx) : m_bmp(bmp), m_idx(idx) {} public: Iterator& operator++() {++m_idx; return *this;} bool operator*() {return m_bmp.getBit(m_idx);} bool operator!=(const Iterator& other) const {return m_idx != other.m_idx;} }; Iterator begin() const {return Iterator(*this, 0);} Iterator end() const {return Iterator(*this, m_bitCount);} }; /* PAK entry stream reader */ class PAKEntryReadStream : public Athena::io::IStreamReader { std::unique_ptr m_buf; atUint64 m_sz; atUint64 m_pos; public: PAKEntryReadStream() {} operator bool() const {return m_buf.operator bool();} PAKEntryReadStream(const PAKEntryReadStream& other) = delete; PAKEntryReadStream(PAKEntryReadStream&& other) = default; PAKEntryReadStream& operator=(const PAKEntryReadStream& other) = delete; PAKEntryReadStream& operator=(PAKEntryReadStream&& other) = default; PAKEntryReadStream(std::unique_ptr&& buf, atUint64 sz, atUint64 pos) : m_buf(std::move(buf)), m_sz(sz), m_pos(pos) { if (m_pos >= m_sz) LogDNACommon.report(LogVisor::FatalError, "PAK stream cursor overrun"); } inline void seek(atInt64 pos, Athena::SeekOrigin origin) { if (origin == Athena::Begin) m_pos = pos; else if (origin == Athena::Current) m_pos += pos; else if (origin == Athena::End) m_pos = m_sz + pos; if (m_pos >= m_sz) LogDNACommon.report(LogVisor::FatalError, "PAK stream cursor overrun"); } inline atUint64 position() const {return m_pos;} inline atUint64 length() const {return m_sz;} inline const atUint8* data() const {return m_buf.get();} inline atUint64 readUBytesToBuf(void* buf, atUint64 len) { atUint64 bufEnd = m_pos + len; if (bufEnd > m_sz) len -= bufEnd - m_sz; memcpy(buf, m_buf.get() + m_pos, len); m_pos += len; return len; } }; struct UniqueResult { enum Type { UNIQUE_NOTFOUND, UNIQUE_LEVEL, UNIQUE_AREA, UNIQUE_LAYER } type = UNIQUE_NOTFOUND; const HECL::SystemString* areaName = nullptr; const HECL::SystemString* layerName = nullptr; UniqueResult() = default; UniqueResult(Type tp) : type(tp) {} inline HECL::ProjectPath uniquePath(const HECL::ProjectPath& pakPath) const { if (type == UNIQUE_AREA) { HECL::ProjectPath areaDir(pakPath, *areaName); areaDir.makeDir(); return areaDir; } else if (type == UNIQUE_LAYER) { HECL::ProjectPath areaDir(pakPath, *areaName); areaDir.makeDir(); HECL::ProjectPath layerDir(areaDir, *layerName); layerDir.makeDir(); return layerDir; } return pakPath; } }; template class PAKRouter; /* Resource extractor type */ template struct ResExtractor { std::function func_a; std::function&, const typename PAKBRIDGE::PAKType::Entry&, bool)> func_b; const char* fileExts[4]; unsigned weight; }; /* PAKRouter (for detecting shared entry locations) */ template class PAKRouter { public: using PAKType = typename BRIDGETYPE::PAKType; using IDType = typename PAKType::IDType; using EntryType = typename PAKType::Entry; private: const SpecBase& m_dataSpec; const std::vector* m_bridges = nullptr; std::vector> m_bridgePaths; size_t m_curBridgeIdx = 0; const HECL::ProjectPath& m_gameWorking; const HECL::ProjectPath& m_gameCooked; HECL::ProjectPath m_sharedWorking; HECL::ProjectPath m_sharedCooked; const PAKType* m_pak = nullptr; const NOD::DiscBase::IPartition::Node* m_node = nullptr; std::unordered_map> m_uniqueEntries; std::unordered_map> m_sharedEntries; public: PAKRouter(const SpecBase& dataSpec, const HECL::ProjectPath& working, const HECL::ProjectPath& cooked) : m_dataSpec(dataSpec), m_gameWorking(working), m_gameCooked(cooked), m_sharedWorking(working, "Shared"), m_sharedCooked(cooked, "Shared") {} void build(std::vector& bridges, std::function progress) { m_bridges = &bridges; m_bridgePaths.clear(); m_uniqueEntries.clear(); m_sharedEntries.clear(); size_t count = 0; float bridgesSz = bridges.size(); /* Route entries unique/shared per-pak */ size_t bridgeIdx = 0; for (BRIDGETYPE& bridge : bridges) { const std::string& name = bridge.getName(); HECL::SystemStringView sysName(name); HECL::SystemString::const_iterator extit = sysName.sys_str().end() - 4; HECL::SystemString baseName(sysName.sys_str().begin(), extit); m_bridgePaths.emplace_back(std::make_pair(HECL::ProjectPath(m_gameWorking, baseName), HECL::ProjectPath(m_gameCooked, baseName))); bridge.build(); const typename BRIDGETYPE::PAKType& pak = bridge.getPAK(); for (const auto& entry : pak.m_idMap) { auto sSearch = m_sharedEntries.find(entry.first); if (sSearch != m_sharedEntries.end()) continue; auto uSearch = m_uniqueEntries.find(entry.first); if (uSearch != m_uniqueEntries.end()) { m_uniqueEntries.erase(uSearch); m_sharedEntries[entry.first] = std::make_pair(bridgeIdx, entry.second); } else m_uniqueEntries[entry.first] = std::make_pair(bridgeIdx, entry.second); } progress(++count / bridgesSz); ++bridgeIdx; } } void enterPAKBridge(const BRIDGETYPE& pakBridge) { auto pit = m_bridgePaths.begin(); size_t bridgeIdx = 0; for (const BRIDGETYPE& bridge : *m_bridges) { if (&bridge == &pakBridge) { pit->first.makeDir(); pit->second.makeDir(); m_pak = &pakBridge.getPAK(); m_node = &pakBridge.getNode(); m_curBridgeIdx = bridgeIdx; return; } ++pit; ++bridgeIdx; } LogDNACommon.report(LogVisor::FatalError, "PAKBridge provided to PAKRouter::enterPAKBridge() was not part of build()"); } HECL::ProjectPath getWorking(const typename BRIDGETYPE::PAKType::Entry* entry, const ResExtractor& extractor) const { if (!m_pak) LogDNACommon.report(LogVisor::FatalError, "PAKRouter::enterPAKBridge() must be called before PAKRouter::getWorkingPath()"); auto uniqueSearch = m_uniqueEntries.find(entry->id); if (uniqueSearch != m_uniqueEntries.end()) { const HECL::ProjectPath& pakPath = m_bridgePaths[uniqueSearch->second.first].first; pakPath.makeDir(); HECL::ProjectPath uniquePath = entry->unique.uniquePath(pakPath); HECL::SystemString entName = m_pak->bestEntryName(*entry); if (extractor.fileExts[0] && !extractor.fileExts[1]) entName += extractor.fileExts[0]; return HECL::ProjectPath(uniquePath, entName); } auto sharedSearch = m_sharedEntries.find(entry->id); if (sharedSearch != m_sharedEntries.end()) { const HECL::ProjectPath& pakPath = m_bridgePaths[m_curBridgeIdx].first; HECL::ProjectPath uniquePathPre = entry->unique.uniquePath(pakPath); HECL::SystemString entBase = m_pak->bestEntryName(*entry); HECL::SystemString entName = entBase; if (extractor.fileExts[0] && !extractor.fileExts[1]) entName += extractor.fileExts[0]; HECL::ProjectPath sharedPath(m_sharedWorking, entName); HECL::ProjectPath uniquePath(uniquePathPre, entName); if (extractor.func_a || extractor.func_b) { if (extractor.fileExts[0] && !extractor.fileExts[1]) uniquePath.makeLinkTo(sharedPath); else { for (int e=0 ; e<4 ; ++e) { if (!extractor.fileExts[e]) break; HECL::SystemString entName = entBase + extractor.fileExts[e]; HECL::ProjectPath sharedPath(m_sharedWorking, entName); HECL::ProjectPath uniquePath(uniquePathPre, entName); uniquePath.makeLinkTo(sharedPath); } } } m_sharedWorking.makeDir(); return sharedPath; } LogDNACommon.report(LogVisor::FatalError, "Unable to find entry %s", entry->id.toString().c_str()); return HECL::ProjectPath(); } HECL::ProjectPath getWorking(const typename BRIDGETYPE::PAKType::Entry* entry) const { return getWorking(entry, BRIDGETYPE::LookupExtractor(*entry)); } HECL::ProjectPath getWorking(const typename BRIDGETYPE::PAKType::IDType& id) const { return getWorking(lookupEntry(id)); } HECL::ProjectPath getCooked(const typename BRIDGETYPE::PAKType::Entry* entry) const { if (!m_pak) LogDNACommon.report(LogVisor::FatalError, "PAKRouter::enterPAKBridge() must be called before PAKRouter::getCookedPath()"); auto uniqueSearch = m_uniqueEntries.find(entry->id); if (uniqueSearch != m_uniqueEntries.end()) { const HECL::ProjectPath& pakPath = m_bridgePaths[uniqueSearch->second.first].second; pakPath.makeDir(); HECL::ProjectPath uniquePath = entry->unique.uniquePath(pakPath); return HECL::ProjectPath(uniquePath, m_pak->bestEntryName(*entry)); } auto sharedSearch = m_sharedEntries.find(entry->id); if (sharedSearch != m_sharedEntries.end()) { m_sharedCooked.makeDir(); return HECL::ProjectPath(m_sharedCooked, m_pak->bestEntryName(*entry)); } LogDNACommon.report(LogVisor::FatalError, "Unable to find entry %s", entry->id.toString().c_str()); return HECL::ProjectPath(); } HECL::ProjectPath getCooked(const typename BRIDGETYPE::PAKType::IDType& id) const { return getCooked(lookupEntry(id)); } HECL::SystemString getResourceRelativePath(const typename BRIDGETYPE::PAKType::Entry& a, const typename BRIDGETYPE::PAKType::IDType& b) const { if (!m_pak) LogDNACommon.report(LogVisor::FatalError, "PAKRouter::enterPAKBridge() must be called before PAKRouter::getResourceRelativePath()"); const typename BRIDGETYPE::PAKType::Entry* be = lookupEntry(b); if (!be) return HECL::SystemString(); HECL::ProjectPath aPath = getWorking(&a, BRIDGETYPE::LookupExtractor(a)); HECL::SystemString ret; for (int i=0 ; ibestEntryName(entry); } std::string getBestEntryName(const typename BRIDGETYPE::PAKType::IDType& entry) const { if (!m_pak) LogDNACommon.report(LogVisor::FatalError, "PAKRouter::enterPAKBridge() must be called before PAKRouter::getBestEntryName()"); const typename BRIDGETYPE::PAKType::Entry* e = m_pak->lookupEntry(entry); if (!e) return entry.toString(); return m_pak->bestEntryName(*e); } bool extractResources(const BRIDGETYPE& pakBridge, bool force, std::function progress) { enterPAKBridge(pakBridge); size_t count = 0; size_t sz = m_pak->m_idMap.size(); float fsz = sz; for (unsigned w=0 ; countm_idMap) { ResExtractor extractor = BRIDGETYPE::LookupExtractor(*item.second); if (extractor.weight != w) continue; HECL::ProjectPath cooked = getCooked(item.second); if (force || cooked.getPathType() == HECL::ProjectPath::PT_NONE) { PAKEntryReadStream s = item.second->beginReadStream(*m_node); FILE* fout = HECL::Fopen(cooked.getAbsolutePath().c_str(), _S("wb")); fwrite(s.data(), 1, s.length(), fout); fclose(fout); } HECL::ProjectPath working = getWorking(item.second, extractor); if (extractor.func_a) /* Doesn't need PAKRouter access */ { if (force || working.getPathType() == HECL::ProjectPath::PT_NONE) { PAKEntryReadStream s = item.second->beginReadStream(*m_node); extractor.func_a(m_dataSpec, s, working); } } else if (extractor.func_b) /* Needs PAKRouter access */ { if (force || working.getPathType() == HECL::ProjectPath::PT_NONE) { PAKEntryReadStream s = item.second->beginReadStream(*m_node); extractor.func_b(m_dataSpec, s, working, *this, *item.second, force); } } progress(++count / fsz); } } return true; } const typename BRIDGETYPE::PAKType::Entry* lookupEntry(const typename BRIDGETYPE::PAKType::IDType& entry, const NOD::DiscBase::IPartition::Node** nodeOut=nullptr) const { if (!m_bridges) LogDNACommon.report(LogVisor::FatalError, "PAKRouter::build() must be called before PAKRouter::lookupEntry()"); if (m_pak) { const typename BRIDGETYPE::PAKType::Entry* ent = m_pak->lookupEntry(entry); if (ent) { if (nodeOut) *nodeOut = m_node; return ent; } } for (const BRIDGETYPE& bridge : *m_bridges) { const typename BRIDGETYPE::PAKType& pak = bridge.getPAK(); const typename BRIDGETYPE::PAKType::Entry* ent = pak.lookupEntry(entry); if (ent) { if (nodeOut) *nodeOut = &bridge.getNode(); return ent; } } LogDNACommon.report(LogVisor::Warning, "unable to find PAK entry %s", entry.toString().c_str()); if (nodeOut) *nodeOut = nullptr; return nullptr; } template bool lookupAndReadDNA(const typename BRIDGETYPE::PAKType::IDType& id, DNA& out) { const NOD::DiscBase::IPartition::Node* node; const typename BRIDGETYPE::PAKType::Entry* entry = lookupEntry(id, &node); if (!entry) return false; PAKEntryReadStream rs = entry->beginReadStream(*node); out.read(rs); return true; } }; /* Resource cooker function */ typedef std::function ResCooker; } /* Hash template-specializations for UniqueID types */ namespace std { template<> struct hash { inline size_t operator()(const Retro::FourCC& fcc) const {return fcc.toUint32();} }; template<> struct hash { inline size_t operator()(const Retro::UniqueID32& id) const {return id.toUint32();} }; template<> struct hash { inline size_t operator()(const Retro::UniqueID64& id) const {return id.toUint64();} }; template<> struct hash { inline size_t operator()(const Retro::UniqueID128& id) const {return id.toHighUint64() ^ id.toLowUint64();} }; } #endif // __DNA_COMMON_HPP__